def build_model(args):
# build model
if args.model == 'cnn' and args.dataset == 'cifar':
net_glob = CNNCifar(args=args).to(args.device)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_glob = CNNMnist(args=args).to(args.device)
elif args.model == 'LeNet' and args.dataset == 'traffic':
net_glob = LeNet(args=args).to(args.device)
else:
exit('Error: unrecognized model')
return net_glob
def get_model(args):
if args.model == 'cnn' and args.dataset in ['cifar10', 'cifar100']:
net_glob = CNNCifar(args=args).to(args.device)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_glob = CNNMnist(args=args).to(args.device)
elif args.model == 'mlp' and args.dataset == 'mnist':
net_glob = MLP(dim_in=784, dim_hidden=256,
dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
print(net_glob)
return net_glob
def __init__(self, args):
if args.dataset == 'cifar':
self.net = CNNCifar(args=args).to(args.device)
else:
self.net = CNNMnist(args=args).to(args.device)
self.net.train()
self.loss_func = nn.CrossEntropyLoss()
self.optimizer = torch.optim.SGD(self.net.parameters(), lr=args.lr)
self.args = args
self.w_glob = []
# key exchange
self.x = self.gx = 0
self.keys = defaultdict(int)
def build_model():
# build model
if args.model == 'cnn' and args.dataset == 'cifar':
net_glob = CNNCifar(args=args).to(args.device)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_glob = CNNMnist(args=args).to(args.device)
# elif args.model == 'mlp':
# len_in = 1
# for x in img_size:
# len_in *= x
# net_glob = MLP(dim_in=len_in, dim_hidden=200, dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
return net_glob
def get_model(args):
if args.model == 'cnn' and args.dataset in ['cifar10', 'cifar100']:
net_glob = CNNCifar(args=args).to(args.device)
elif args.model == 'mobile' and args.dataset in ['cifar10', 'cifar100']:
net_glob = MobileNetCifar(num_classes=args.num_classes).to(args.device)
elif args.model == 'resnet18' and args.dataset in ['cifar10', 'cifar100']:
net_glob = ResNet18(num_classes=args.num_classes).to(args.device)
elif args.model == 'resnet50' and args.dataset in ['cifar10', 'cifar100']:
net_glob = ResNet50(num_classes=args.num_classes).to(args.device)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_glob = CNNMnist(args=args).to(args.device)
elif args.model == 'mlp' and args.dataset == 'mnist':
net_glob = MLP(dim_in=784, dim_hidden=256,
dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
return net_glob
def modelBuild():
"""
Build the basic training network and return the related args.
"""
# build model
args = args_parser()
args.device = torch.device('cuda:{}'.format(
args.gpu) if torch.cuda.is_available() and args.gpu != -1 else 'cpu')
# load dataset and split users
if args.dataset == 'mnist':
trans_mnist = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])
dataset_train = datasets.MNIST('../data/mnist/',
train=True,
download=True,
transform=trans_mnist)
dataset_test = datasets.MNIST('../data/mnist/',
train=False,
download=True,
transform=trans_mnist)
# sample users
if args.iid:
# allocate the dataset index to users
dict_users = mnist_iid(dataset_train, args.num_users)
else:
dict_users = mnist_noniid(dataset_train, args.num_users)
elif args.dataset == 'cifar':
trans_cifar = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
dataset_train = datasets.CIFAR10('../data/cifar',
train=True,
download=True,
transform=trans_cifar)
dataset_test = datasets.CIFAR10('../data/cifar',
train=False,
download=True,
transform=trans_cifar)
if args.iid:
dict_users = cifar_iid(dataset_train, args.num_users)
else:
exit('Error: only consider IID setting in CIFAR10')
else:
exit('Error: unrecognized dataset')
print("The para of iid is " + str(args.iid))
img_size = dataset_train[0][0].shape
if args.model == 'cnn' and args.dataset == 'cifar':
net_glob = CNNCifar(args=args).to(args.device)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_glob = CNNMnist(args=args).to(args.device)
elif args.model == 'mlp':
len_in = 1
for x in img_size:
len_in *= x
net_glob = MLP(dim_in=len_in, dim_hidden=200,
dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
print("********************************")
print(net_glob)
print("********************************")
return net_glob, args, dataset_train, dataset_test, dict_users
transform=transform)
test_set = datasets.CIFAR10('./data/cifar',
train=False,
download=False,
transform=transform)
else:
exit('Error: unrecognized dataset...')
# split dataset {user_id: [list of data index]}
dict_users_train, ratio = noniid_train(train_set, args.num_users)
dict_users_test = noniid_test(test_set, args.num_users, ratio)
print('Data finished...')
# load global model
net_glob = CNNCifar(
args=args).to(args.device) if args.dataset == 'cifar' else CNNMnist(
args=args).to(args.device)
net_glob.train()
# parameters
w_glob = net_glob.state_dict()
loss_train = []
# meta-learning for global initial parameters
for epoch in range(args.meta_epochs):
loss_locals = []
w_locals = []
m = max(int(args.frac * args.num_users), 1)
idxs_users = np.random.choice(range(args.num_users), m, replace=False)
for idx in idxs_users:
dp.gen_local_imbalance(10, 5000, 0.8)
imbalanced_way = "local"
elif args.global_balance:
dp.gen_global_imbalance(
5, 2000, [500, 500, 1000, 1000, 1500, 1500, 3000, 1000, 0, 0])
imbalanced_way = "global"
# without self-balanced
db = DataBalance.DataBalance(dp)
db.assign_clients(False)
# load dataset and split users
img_size = dp[0][0].shape
# build original model
net_glob = None
if args.model == 'cnn' and args.dataset == 'cifar':
net_glob = CNNCifar(args=args).to(args.device)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_glob = CNNMnist(args=args).to(args.device)
elif args.model == 'mlp':
len_in = 1
for x in img_size:
len_in *= x
net_glob = MLP(dim_in=len_in, dim_hidden=200,
dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
print(net_glob)
net_glob.train()
# copy weights
w_glob = net_glob.state_dict()
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
dataset_train = datasets.CIFAR10('./data/cifar',
train=True,
transform=transform,
target_transform=None,
download=True)
img_size = dataset_train[0][0].shape
else:
exit('Error: unrecognized dataset')
# build model
if args.model == 'cnn' and args.dataset == 'cifar':
net_glob = CNNCifar(args=args).to(args.device)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_glob = CNNMnist(args=args).to(args.device)
elif args.model == 'mlp':
len_in = 1
for x in img_size:
len_in *= x
net_glob = MLP(dim_in=len_in, dim_hidden=64,
dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
print(net_glob)
# training
optimizer = optim.SGD(net_glob.parameters(),
lr=args.lr,
]))
# testing
dataset_test = datasets.FashionMNIST('../data/fmnist/', train=False, download=True,
transform=transforms.Compose([
transforms.Resize((32, 32)),
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
]))
test_loader = DataLoader(dataset_test, batch_size=1000, shuffle=False)
else:
exit('Error: unrecognized dataset')
# build model
if args.model == 'lenet' and (args.dataset == 'cifar' or args.dataset == 'fmnist'):
net_glob = CNNCifar(args=args).to(args.device)
elif args.model == 'vgg' and args.dataset == 'cifar':
net_glob = vgg16().to(args.device)
else:
exit('Error: unrecognized model')
print(net_glob)
img = dataset_train[0][0].unsqueeze(0).to(args.device)
writer.add_graph(net_glob, img)
# training
creterion = nn.CrossEntropyLoss()
train_loader = DataLoader(dataset_train, batch_size=64, shuffle=True)
# optimizer = optim.Adam(net_glob.parameters())
optimizer = optim.SGD(net_glob.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4)
# scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=40, gamma=0.1)
# # # scheduler.step()
train_set = datasets.MNIST(root='./data/mnist', train=True, download=False, transform=transform)
test_set = datasets.MNIST(root='./data/mnist', train=False, download=False, transform=transform)
elif args.dataset == 'cifar':
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
train_set = datasets.CIFAR10('./data/cifar', train=True, download=False, transform=transform)
test_set = datasets.CIFAR10('./data/cifar', train=False, download=False, transform=transform)
else:
exit('Error: unrecognized dataset...')
dict_users_train, ratio = noniid_train2(train_set, args.num_users)
dict_users_test = noniid_test(test_set, args.num_users, ratio)
print('Data finished...')
# load global model
net_glob = CNNCifar(args=args).to(args.device) if args.dataset == 'cifar' else CNNMnist(args=args).to(args.device)
net_glob.train()
# parameters
w_glob = net_glob.state_dict()
# test each of clients
test_acc = [0 for i in range(args.num_users)]
test_loss = [0 for i in range(args.num_users)]
for idx in range(args.num_users):
# every time start with the same global parameters
net_glob.load_state_dict(w_glob)
client = Client(args=args, dataset=train_set, idxs=dict_users_train[idx], bs=args.train_bs)
w_client = client.local_train(net=copy.deepcopy(net_glob).to(args.device))
client = Client(args=args, dataset=test_set, idxs=dict_users_test[idx], bs=args.test_bs)
def main():
# parse args
args = args_parser()
args.device = torch.device('cuda:{}'.format(
args.gpu) if torch.cuda.is_available() and args.gpu != -1 else 'cpu')
# load dataset and split users
if args.dataset == 'mnist':
trans_mnist = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])
dataset_train = datasets.MNIST('../data/mnist/',
train=True,
download=True,
transform=trans_mnist)
dataset_test = datasets.MNIST('../data/mnist/',
train=False,
download=True,
transform=trans_mnist)
print("type of test dataset", type(dataset_test))
# sample users
if args.iid:
dict_users = mnist_iid(dataset_train, args.num_users)
else:
dict_users, dict_labels_counter = mnist_noniid(
dataset_train, args.num_users)
dict_users_2, dict_labels_counter_2 = dict_users, dict_labels_counter
#dict_users, dict_labels_counter = mnist_noniid_unequal(dataset_train, args.num_users)
elif args.dataset == 'cifar':
trans_cifar = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
dataset_train = datasets.CIFAR10('../data/cifar',
train=True,
download=True,
transform=trans_cifar)
dataset_test = datasets.CIFAR10('../data/cifar',
train=False,
download=True,
transform=trans_cifar)
if args.iid:
dict_users = cifar_iid(dataset_train, args.num_users)
else:
exit('Error: only consider IID setting in CIFAR10')
else:
exit('Error: unrecognized dataset')
img_size = dataset_train[0][0].shape
# build model
if args.model == 'cnn' and args.dataset == 'cifar':
net_glob = CNNCifar(args=args).to(args.device)
net_glob_2 = CNNCifar(args=args).to(args.device)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_glob = CNNMnist(args=args).to(args.device)
net_glob_2 = CNNMnist(args=args).to(args.device)
elif args.model == 'mlp':
len_in = 1
for x in img_size:
len_in *= x
net_glob = MLP(dim_in=len_in, dim_hidden=200,
dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
#print(net_glob)
#net_glob.train()
acc_test, loss_test = test_img(net_glob, dataset_test, args)
print("val test finished")
print("{:.2f}".format(acc_test))
temp = net_glob
#net_glob_2 = net_glob
temp_2 = net_glob_2
# copy weights
w_glob = net_glob.state_dict()
# training
loss_train = []
cv_loss, cv_acc = [], []
val_loss_pre, counter = 0, 0
net_best = None
best_loss = None
val_acc_list, net_list = [], []
Loss_local_each_global_total = []
test_ds, valid_ds = torch.utils.data.random_split(dataset_test,
(9500, 500))
loss_workers_total = np.zeros(shape=(args.num_users, args.epochs))
label_workers = {
i: np.array([], dtype='int64')
for i in range(args.num_users)
}
workers_percent = []
workers_count = 0
acc_test_global, loss_test_global = test_img(x, valid_ds, args)
selected_users_index = []
for idx in range(args.num_users):
# print("train started")
local = LocalUpdate(args=args,
dataset=dataset_train,
idxs=dict_users[idx])
w, loss = local.train(net=copy.deepcopy(net_glob).to(args.device))
# print(w)
# print("train completed")
# temp = FedAvg(w)
temp.load_state_dict(w)
temp.eval()
acc_test_local, loss_test_local = test_img(temp, valid_ds, args)
loss_workers_total[idx, iter] = acc_test_local
if workers_count >= (args.num_users / 2):
break
elif acc_test_local >= (0.7 * acc_test_global):
selected_users_index.append(idx)
for iter in range(args.epochs):
print("round started")
Loss_local_each_global = []
loss_workers = np.zeros((args.num_users, args.epochs))
w_locals, loss_locals = [], []
m = max(int(args.frac * args.num_users), 1)
#idxs_users = np.random.choice(range(args.num_users), m, replace=False)
#if iter % 5 == 0:
# Minoo
x = net_glob
x.eval()
Loss_local_each_global_total.append(acc_test_global)
for idx in selected_users_index:
#print("train started")
local = LocalUpdate(args=args,
dataset=dataset_train,
idxs=dict_users[idx])
w, loss = local.train(net=copy.deepcopy(net_glob).to(args.device))
#print(w)
#print("train completed")
#temp = FedAvg(w)
temp.load_state_dict(w)
temp.eval()
acc_test_local, loss_test_local = test_img(temp, valid_ds, args)
loss_workers_total[idx, iter] = acc_test_local
if workers_count >= (args.num_users / 2):
break
elif acc_test_local >= (0.7 * acc_test_global):
w_locals.append(copy.deepcopy(w))
loss_locals.append(copy.deepcopy(loss))
print("Update Received")
workers_count += 1
# update global weights
w_glob = FedAvg(w_locals)
# copy weight to net_glob
net_glob.load_state_dict(w_glob)
print("round completed")
loss_avg = sum(loss_locals) / len(loss_locals)
print('Round {:3d}, Average loss {:.3f}'.format(iter, loss_avg))
loss_train.append(loss_avg)
workers_percent.append(workers_count)
# plot loss curve
plt.figure()
plt.plot(range(len(workers_percent)), workers_percent)
plt.ylabel('train_loss')
plt.savefig(
'./save/Newfed_WorkersPercent_0916_{}_{}_{}_C{}_iid{}.png'.format(
args.dataset, args.model, args.epochs, args.frac, args.iid))
# print(loss_workers_total)
# plot loss curve
# plt.figure()
# plt.plot(range(len(loss_train)), loss_train)
# plt.ylabel('train_loss')
# plt.savefig('./save/Newfed_0916_{}_{}_{}_C{}_iid{}.png'.format(args.dataset, args.model, args.epochs, args.frac, args.iid))
#
plt.figure()
for i in range(args.num_users):
plot = plt.plot(range(len(loss_workers_total[i, :])),
loss_workers_total[i, :],
label="Worker {}".format(i))
plot5 = plt.plot(range(len(Loss_local_each_global_total)),
Loss_local_each_global_total,
color='000000',
label="Global")
plt.legend(loc='best')
plt.ylabel('Small Test Set Accuracy of workers')
plt.xlabel('Number of Rounds')
plt.savefig(
'./save/NewFed_2workers_Acc_0916_{}_{}_{}_C{}_iid{}.png'.format(
args.dataset, args.model, args.epochs, args.frac, args.iid))
# plt.figure()
# bins = np.linspace(0, 9, 3)
# a = dict_labels_counter[:, 0].ravel()
# print(type(a))
# b = dict_labels_counter[:, 1].ravel()
# x_labels = ['0', '1', '2', '3','4','5','6','7','8','9']
# # Set plot parameters
# fig, ax = plt.subplots()
# width = 0.1 # width of bar
# x = np.arange(10)
# ax.bar(x, dict_labels_counter[:, 0], width, color='#000080', label='Worker 1')
# ax.bar(x + width, dict_labels_counter[:, 1], width, color='#73C2FB', label='Worker 2')
# ax.bar(x + 2*width, dict_labels_counter[:, 2], width, color='#ff0000', label='Worker 3')
# ax.bar(x + 3*width, dict_labels_counter[:, 3], width, color='#32CD32', label='Worker 4')
# ax.set_ylabel('Number of Labels')
# ax.set_xticks(x + width + width / 2)
# ax.set_xticklabels(x_labels)
# ax.set_xlabel('Labels')
# ax.legend()
# plt.grid(True, 'major', 'y', ls='--', lw=.5, c='k', alpha=.3)
# fig.tight_layout()
# plt.savefig(
# './save/Newfed_2workersLabels_0916_{}_{}_{}_C{}_iid{}.png'.format(args.dataset, args.model, args.epochs, args.frac,
# args.iid))
# testing
print("testing started")
net_glob.eval()
print("train test started")
acc_train_final, loss_train_final = test_img(net_glob, dataset_train, args)
print("train test finished")
acc_test_final, loss_test_final = test_img(net_glob, dataset_test, args)
print("val test finished")
#print("Training accuracy: {:.2f}".format(acc_train))
#print("Testing accuracy: {:.2f}".format(acc_test))
print("{:.2f}".format(acc_test_final))
#print("{:.2f".format(Loss_local_each_worker))
# training
w_glob_2 = net_glob_2.state_dict()
loss_train_2 = []
cv_loss_2, cv_acc_2 = [], []
val_loss_pre_2, counter_2 = 0, 0
net_best_2 = None
best_loss_2 = None
val_acc_list_2, net_list_2 = [], []
Loss_local_each_global_total_2 = []
loss_workers_total_2 = np.zeros(shape=(args.num_users, args.epochs))
label_workers_2 = {
i: np.array([], dtype='int64')
for i in range(args.num_users)
}
for iter in range(args.epochs):
print("round started")
Loss_local_each_global_2 = []
loss_workers_2 = np.zeros((args.num_users, args.epochs))
w_locals_2, loss_locals_2 = [], []
m_2 = max(int(args.frac * args.num_users), 1)
idxs_users_2 = np.random.choice(range(args.num_users),
m_2,
replace=False)
# Minoo
x_2 = net_glob_2
x_2.eval()
acc_test_global_2, loss_test_global_2 = test_img(x_2, valid_ds, args)
Loss_local_each_global_total_2.append(acc_test_global_2)
for idx in idxs_users_2:
#print("train started")
local_2 = LocalUpdate(args=args,
dataset=dataset_train,
idxs=dict_users_2[idx])
w_2, loss_2 = local_2.train(
net=copy.deepcopy(net_glob_2).to(args.device))
#print(w)
#print("train completed")
w_locals_2.append(copy.deepcopy(w_2))
loss_locals_2.append(copy.deepcopy(loss_2))
#temp = FedAvg(w)
temp_2.load_state_dict(w_2)
temp_2.eval()
acc_test_local_2, loss_test_local_2 = test_img(
temp_2, valid_ds, args)
loss_workers_total_2[idx, iter] = acc_test_local_2
# update global weights
w_glob_2 = FedAvg(w_locals_2)
# copy weight to net_glob
net_glob_2.load_state_dict(w_glob_2)
print("round completed")
loss_avg_2 = sum(loss_locals_2) / len(loss_locals_2)
print('Round {:3d}, Average loss {:.3f}'.format(iter, loss_avg_2))
loss_train_2.append(loss_avg_2)
print("round completed")
# plot loss curve
plt.figure()
plt.plot(range(len(loss_train_2)),
loss_train_2,
color='#000000',
label="Main FL")
plt.plot(range(len(loss_train)),
loss_train,
color='#ff0000',
label="Centralized Algorithm")
plt.ylabel('train_loss')
plt.savefig('./save/main_fed_0916_{}_{}_{}_C{}_iid{}.png'.format(
args.dataset, args.model, args.epochs, args.frac, args.iid))
# print(loss_workers_total)
plt.figure()
for i in range(args.num_users):
plot = plt.plot(range(len(loss_workers_total_2[i, :])),
loss_workers_total_2[i, :],
label="Worker {}".format(i))
plot5 = plt.plot(range(len(Loss_local_each_global_total_2)),
Loss_local_each_global_total_2,
color='000000',
label="Global")
plt.legend(loc='best')
plt.ylabel('Small Test Set Accuracy of workers')
plt.xlabel('Number of Rounds')
plt.savefig('./save/mainfed_Acc_0916_{}_{}_{}_C{}_iid{}.png'.format(
args.dataset, args.model, args.epochs, args.frac, args.iid))
# plt.figure()
# bins = np.linspace(0, 9, 3)
# a = dict_labels_counter_2[:, 0].ravel()
# print(type(a))
# b = dict_labels_counter_2[:, 1].ravel()
# x_labels = ['0', '1', '2', '3','4','5','6','7','8','9']
# # Set plot parameters
# fig, ax = plt.subplots()
# width = 0.1 # width of bar
# x = np.arange(10)
# ax.bar(x, dict_labels_counter_2[:, 0], width, color='#000080', label='Worker 1')
# ax.bar(x + width, dict_labels_counter_2[:, 1], width, color='#73C2FB', label='Worker 2')
# ax.bar(x + 2*width, dict_labels_counter_2[:, 2], width, color='#ff0000', label='Worker 3')
# ax.bar(x + 3*width, dict_labels_counter_2[:, 3], width, color='#32CD32', label='Worker 4')
# ax.set_ylabel('Number of Labels')
# ax.set_xticks(x + width + width / 2)
# ax.set_xticklabels(x_labels)
# ax.set_xlabel('Labels')
# ax.legend()
# plt.grid(True, 'major', 'y', ls='--', lw=.5, c='k', alpha=.3)
# fig.tight_layout()
# plt.savefig(
# './save/main_fed_2workersLabels_0916_{}_{}_{}_C{}_iid{}.png'.format(args.dataset, args.model, args.epochs, args.frac,
# args.iid))
# testing
print("testing started")
net_glob.eval()
print("train test started")
acc_train_final, loss_train_final = test_img(net_glob, dataset_train, args)
print("train test finished")
acc_test_final, loss_test_final = test_img(net_glob, dataset_test, args)
print("val test finished")
#print("Training accuracy: {:.2f}".format(acc_train))
#print("Testing accuracy: {:.2f}".format(acc_test))
print("{:.2f}".format(acc_test_final))
#print("{:.2f".format(Loss_local_each_worker))
return loss_test_final, loss_train_final
for k, v in dict_users.items():
writer.add_histogram(f'user_{k}/data_distribution',
np.array(dataset_train.targets)[v],
bins=np.arange(11))
writer.add_histogram(f'all_user/data_distribution',
np.array(dataset_train.targets)[v],
bins=np.arange(11),
global_step=k)
test_loader = DataLoader(dataset_test, batch_size=1000, shuffle=False)
img_size = dataset_train[0][0].shape
# build model
if args.model == 'lenet' and (args.dataset == 'cifar'
or args.dataset == 'fmnist'):
net_glob = CNNCifar(args=args).to(args.device)
elif args.model == 'vgg' and args.dataset == 'cifar':
net_glob = vgg16().to(args.device)
else:
exit('Error: unrecognized model')
print(net_glob)
net_glob.train()
# copy weights
w_init = copy.deepcopy(net_glob.state_dict())
local_acc_final = []
total_acc_final = []
local_acc = np.zeros([args.num_users, args.epochs])
total_acc = np.zeros([args.num_users, args.epochs])
train=False,
transform=cifar_transform(is_training=False),
download=True)
#dataset_train = datasets.CIFAR10('../data/cifar', train=True, download=True, transform=trans_cifar)
#dataset_test = datasets.CIFAR10('../data/cifar', train=False, download=True, transform=trans_cifar)
if args.iid:
dict_users = cifar_iid(dataset_train, args.num_users)
else:
exit('Error: only consider IID setting in CIFAR10')
else:
exit('Error: unrecognized dataset')
img_size = dataset_train[0][0].shape
# build model
if args.model == 'cnn' and args.dataset == 'cifar':
net_glob = CNNCifar(args=args).to(args.device)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_glob = CNNMnist(args=args).to(args.device)
elif args.model == 'mlp':
len_in = 1
for x in img_size:
len_in *= x
net_glob = MLP(dim_in=len_in, dim_hidden=200,
dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
#net_glob = ResNet18()
net_best = 0
net_glob = VGG('VGG16')
def main():
# parse args
args = args_parser()
args.device = torch.device('cuda:{}'.format(args.gpu) if torch.cuda.is_available() and args.gpu != -1 else 'cpu')
# load dataset and split users
if args.dataset == 'mnist':
trans_mnist = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])
dataset_train = datasets.MNIST('../data/mnist/', train=True, download=True, transform=trans_mnist)
dataset_test = datasets.MNIST('../data/mnist/', train=False, download=True, transform=trans_mnist)
# sample users
if args.iid:
dict_users = mnist_iid(dataset_train, args.num_users)
else:
dict_users, dict_labels_counter = mnist_noniid(dataset_train, args.num_users)
dict_users_mainFL, dict_labels_counter_mainFL = dict_users, dict_labels_counter
elif args.dataset == 'cifar':
trans_cifar = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
dataset_train = datasets.CIFAR10('../data/cifar', train=True, download=True, transform=trans_cifar)
dataset_test = datasets.CIFAR10('../data/cifar', train=False, download=True, transform=trans_cifar)
if args.iid:
dict_users = cifar_iid(dataset_train, args.num_users)
else:
dict_users, dict_labels_counter = cifar_noniid(dataset_train, args.num_users)
dict_users_mainFL, dict_labels_counter_mainFL = dict_users, dict_labels_counter
elif args.dataset == 'fmnist':
trans_fmnist = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])
dataset_train = datasets.FashionMNIST('../data/fmnist', train=True, download=True, transform=trans_fmnist)
dataset_test = datasets.FashionMNIST('../data/fmnist', train=False, download=True, transform=trans_fmnist)
if args.iid:
dict_users = mnist_iid(dataset_train, args.num_users)
else:
dict_users, dict_labels_counter = mnist_noniid(dataset_train, args.num_users)
dict_users_mainFL, dict_labels_counter_mainFL = dict_users, dict_labels_counter
else:
exit('Error: unrecognized dataset')
img_size = dataset_train[0][0].shape
acc_full_distributed = []
acc_full_main = []
loss_full_ditributed = []
loss_full_main = []
SD_acc_full_distributed = []
SD_acc_full_main = []
SD_loss_full_ditributed = []
SD_loss_full_main = []
workers_percent_full_distributed = []
workers_percent_full_main = []
variable_start = 0.1
variable_end = 1.0
while_counter = 0.1
counter_array = []
Accuracy_Fraction = []
Workers_Fraction = []
accuracy_fraction_each_round_newFL = 0
workers_fraction_each_round_newFL = 0
accuracy_fraction_each_round_mainFL = 0
workers_fraction_each_round_mainFL = 0
data_main = {}
data_DCFL = {}
data_Global_main = {"C": [], "Round":[], "Average Loss Train": [], "Average Loss Test": [], "Accuracy Test": [],
"Workers Number": [], "Large Test Loss":[], "Large Test Accuracy":[]}
data_Global_DCFL = {"C": [], "Round":[], "Average Loss Train": [], "Average Loss Test": [], "Accuracy Test": [],
"Workers Number": [], "Large Test Loss":[], "Large Test Accuracy":[]}
Final_LargeDataSetTest_DCFL = {"C":[], "Test Accuracy":[], "Test Loss":[], "Train Loss":[], "Train Accuracy":[],
"Total Rounds":[]}
Final_LargeDataSetTest_MainFL = {"C":[], "Test Accuracy": [], "Test Loss": [], "Train Loss": [], "Train Accuracy":[]}
# build model
args.frac = variable_start
test_ds, valid_ds_before = torch.utils.data.random_split(dataset_test, (9500, 500))
valid_ds = create_shared_dataset(valid_ds_before, 200)
#while variable_start <= variable_end:
for c_counter in range(1, 11, 3):
if args.model == 'cnn' and args.dataset == 'cifar':
net_glob = CNNCifar(args=args).to(args.device)
net_glob_mainFL = copy.deepcopy(net_glob)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_glob = CNNMnist(args=args).to(args.device)
net_glob_mainFL = copy.deepcopy(net_glob)
elif args.model == 'cnn' and args.dataset == 'fmnist':
net_glob = CNNFashion_Mnist(args=args).to(args.device)
net_glob_mainFL = copy.deepcopy(net_glob)
elif args.model == 'mlp':
len_in = 1
for x in img_size:
len_in *= x
net_glob = MLP(dim_in=len_in, dim_hidden=200, dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
counter_array.append((c_counter/10))
args.frac = (c_counter/10)
######saving index of workers
dict_workers_index = defaultdict(list)
#############Main FL
w_glob_mainFL = net_glob_mainFL.state_dict()
loss_train_mainFL = []
# cv_loss_2, cv_acc_2 = [], []
# val_loss_pre_2, counter_2 = 0, 0
# net_best_2 = None
# best_loss_2 = None
# val_acc_list_2, net_list_2 = [], []
Loss_local_each_global_total_mainFL = []
Accuracy_local_each_global_total_mainFL = []
loss_workers_total_mainFL = np.zeros(shape=(args.num_users, args.epochs))
label_workers_mainFL = {i: np.array([], dtype='int64') for i in range(args.num_users)}
validation_test_mainFed = []
acc_test, loss_test = test_img(net_glob_mainFL, dataset_test, args)
workers_participation_main_fd = np.zeros((args.num_users, args.epochs))
workers_percent_main = []
# for iter in range(args.epochs):
net_glob_mainFL.eval()
acc_test_final_mainFL, loss_test_final_mainFL = test_img(net_glob_mainFL, dataset_test, args)
while_counter_mainFL = loss_test_final_mainFL
iter_mainFL = 0
workers_mainFL = []
for i in range(args.num_users):
workers_mainFL.append(i)
temp_netglob_mainFL = net_glob_mainFL
while iter_mainFL < (args.epochs/2):
data_main['round_{}'.format(iter_mainFL)] = []
# data_Global_main['round_{}'.format(iter)] = []
# print("round started")
Loss_local_each_global_mainFL = []
loss_workers_mainFL = np.zeros((args.num_users, args.epochs))
w_locals_mainFL, loss_locals_mainFL = [], []
m_mainFL = max(int(args.frac * args.num_users), 1)
idxs_users_mainFL = np.random.choice(range(args.num_users), m_mainFL, replace=False)
list_of_random_workers = random.sample(workers_mainFL, m_mainFL)
for i in range(len(list_of_random_workers)):
dict_workers_index[iter_mainFL].append(list_of_random_workers[i])
x_mainFL = net_glob_mainFL
x_mainFL.eval()
acc_test_global_mainFL, loss_test_global_mainFL = test_img(x_mainFL, valid_ds, args)
Loss_local_each_global_total_mainFL.append(loss_test_global_mainFL)
Accuracy_local_each_global_total_mainFL.append(acc_test_global_mainFL)
SD_acc_full_main.append(acc_test_global_mainFL)
SD_loss_full_main.append(loss_test_global_mainFL)
workers_count_mainFL = 0
temp_accuracy = np.zeros(1)
temp_loss_test = np.zeros(1)
temp_loss_train = np.zeros(1)
for idx in list_of_random_workers:
# print("train started")
local_mainFL = LocalUpdate(args=args, dataset=dataset_train, idxs=dict_users_mainFL[idx])
w_mainFL, loss_mainFL = local_mainFL.train(net=copy.deepcopy(net_glob_mainFL).to(args.device))
# print(w)
# print("train completed")
w_locals_mainFL.append(copy.deepcopy(w_mainFL))
loss_locals_mainFL.append(copy.deepcopy(loss_mainFL))
# temp = FedAvg(w)
temp_netglob_mainFL.load_state_dict(w_mainFL)
temp_netglob_mainFL.eval()
print(pnorm_2(temp_netglob_mainFL, 2))
acc_test_local_mainFL, loss_test_local_mainFL = test_img(temp_netglob_mainFL, valid_ds, args)
temp_accuracy[0] = acc_test_local_mainFL
temp_loss_test[0] = loss_test_local_mainFL
temp_loss_train[0] = loss_mainFL
loss_workers_total_mainFL[idx, iter_mainFL] = acc_test_local_mainFL
workers_participation_main_fd[idx][iter_mainFL] = 1
workers_count_mainFL += 1
data_main['round_{}'.format(iter_mainFL)].append({
'C': args.frac,
'User ID': idx,
# 'Local Update': copy.deepcopy(w_mainFL),
'Loss Train': temp_loss_train[0],
'Loss Test': temp_loss_test[0],
'Accuracy': temp_accuracy[0]
})
# update global weights
w_glob_mainFL = FedAvg(w_locals_mainFL)
# copy weight to net_glob
net_glob_mainFL.load_state_dict(w_glob_mainFL)
# print("round completed")
loss_avg_mainFL = sum(loss_locals_mainFL) / len(loss_locals_mainFL)
# print('Round {:3d}, Average loss {:.3f}'.format(iter, loss_avg_mainFL))
loss_train_mainFL.append(loss_avg_mainFL)
# print("round completed")
acc_test_round_mainfed, loss_test_round_mainfed = test_img(net_glob_mainFL, dataset_test, args)
validation_test_mainFed.append(acc_test_round_mainfed)
workers_percent_main.append(workers_count_mainFL / args.num_users)
# plot workers percent of participating
print(iter_mainFL, " round main fl finished")
acc_test_final_mainFL, loss_test_final_mainFL = test_img(net_glob_mainFL, dataset_test, args)
while_counter_mainFL = loss_test_final_mainFL
data_Global_main["Round"].append(iter_mainFL)
data_Global_main["C"].append(args.frac)
data_Global_main["Average Loss Train"].append(float(loss_avg_mainFL))
data_Global_main["Average Loss Test"].append(float(loss_test_global_mainFL))
data_Global_main["Accuracy Test"].append(float(acc_test_global_mainFL))
data_Global_main["Workers Number"].append(float(workers_count_mainFL))
data_Global_main["Large Test Loss"].append(float(loss_test_final_mainFL))
data_Global_main["Large Test Accuracy"].append(float(acc_test_final_mainFL))
iter_mainFL = iter_mainFL + 1
workers_percent_final_mainFL = np.zeros(args.num_users)
workers_name_mainFL = np.empty(args.num_users)
for i in range(len(workers_participation_main_fd[:, 1])):
workers_percent_final_mainFL[i] = sum(workers_participation_main_fd[i, :]) / args.epochs
workers_name_mainFL[i] = i
net_glob_mainFL.eval()
# print("train test started")
acc_train_final_main, loss_train_final_main = test_img(net_glob_mainFL, dataset_train, args)
# print("train test finished")
acc_test_final_main, loss_test_final_main = test_img(net_glob_mainFL, dataset_test, args)
Final_LargeDataSetTest_MainFL["C"].append(args.frac)
Final_LargeDataSetTest_MainFL["Test Loss"].append(float(loss_test_final_main))
Final_LargeDataSetTest_MainFL["Test Accuracy"].append(float(acc_test_final_main))
Final_LargeDataSetTest_MainFL["Train Loss"].append(float(loss_train_final_main))
Final_LargeDataSetTest_MainFL["Train Accuracy"].append(float(acc_train_final_main))
# copy weights
w_glob = net_glob.state_dict()
temp_after = copy.deepcopy(net_glob)
temp_before = copy.deepcopy(net_glob)
# training
loss_train = []
# cv_loss, cv_acc = [], []
# val_loss_pre, counter = 0, 0
# net_best = None
# best_loss = None
# val_acc_list, net_list = [], []
Loss_local_each_global_total = []
# valid_ds = create_shared_dataset(dataset_test, 500)
loss_workers_total = np.zeros(shape=(args.num_users, args.epochs))
label_workers = {i: np.array([], dtype='int64') for i in range(args.num_users)}
workers_percent_dist = []
validation_test_newFed = []
workers_participation = np.zeros((args.num_users, args.epochs))
workers = []
for i in range(args.num_users):
workers.append(i)
counter_threshold_decrease = np.zeros(args.epochs)
Global_Accuracy_Tracker = np.zeros(args.epochs)
Global_Loss_Tracker = np.zeros(args.epochs)
threshold = 0.5
alpha = 0.5 ##decrease parameter
beta = 0.1 ##delta accuracy controller
gamma = 0.5 ##threshold decrease parameter
Goal_Loss = float(loss_test_final_main)
#for iter in range(args.epochs):
net_glob.eval()
acc_test_final, loss_test_final = test_img(net_glob, dataset_test, args)
while_counter = float(loss_test_final)
iter = 0
total_rounds_dcfl = 0
while (while_counter + 0.01) > Goal_Loss and iter <= args.epochs:
data_DCFL['round_{}'.format(iter)] = []
Loss_local_each_global = []
loss_workers = np.zeros((args.num_users, args.epochs))
w_locals, loss_locals = [], []
m = max(int(args.frac * args.num_users), 1)
idxs_users = np.random.choice(range(args.num_users), m, replace=False)
counter_threshold = 0
print(iter, " in dist FL started")
#if iter % 5 == 0:
x = copy.deepcopy(net_glob)
x.eval()
acc_test_global, loss_test_global = test_img(x, valid_ds, args)
Loss_local_each_global_total.append(acc_test_global)
Global_Accuracy_Tracker[iter] = acc_test_global
Global_Loss_Tracker[iter] = loss_test_global
if iter > 0 & (Global_Loss_Tracker[iter-1] - Global_Loss_Tracker[iter] <= beta):
threshold = threshold - gamma
if threshold == 0.0:
threshold = 1.0
print("threshold decreased to", threshold)
workers_count = 0
SD_acc_full_distributed.append(acc_test_global)
SD_loss_full_ditributed.append(loss_test_global)
temp_w_locals = []
temp_workers_loss = np.empty(args.num_users)
temp_workers_accuracy = np.empty(args.num_users)
temp_workers_loss_test = np.empty(args.num_users)
temp_workers_loss_differenc = np.empty(args.num_users)
temp_workers_accuracy_differenc = np.empty(args.num_users)
flag = np.zeros(args.num_users)
list_of_random_workers_newfl = []
if iter < (args.epochs/2):
for key, value in dict_workers_index.items():
# print(value)
if key == iter:
list_of_random_workers_newfl = dict_workers_index[key]
else:
list_of_random_workers_newfl = random.sample(workers, m)
for idx in list_of_random_workers_newfl:
#print("train started")
# before starting train
temp_before = copy.deepcopy(net_glob)
# temp_before.load_state_dict(w)
temp_before.eval()
acc_test_local_before, loss_test_local_before = test_img(temp_before, valid_ds, args)
local = LocalUpdate(args=args, dataset=dataset_train, idxs=dict_users[idx])
w, loss = local.train(net=copy.deepcopy(net_glob).to(args.device))
#print(w)
#print("train completed")
#print("type of idx is ", type(temp_w_locals))
temp_w_locals.append(copy.deepcopy(w))
temp_workers_loss[idx] = copy.deepcopy(loss)
temp_after = copy.deepcopy(net_glob)
temp_after.load_state_dict(w)
temp_after.eval()
acc_test_local_after, loss_test_local_after = test_img(temp_after, valid_ds, args)
loss_workers_total[idx, iter] = loss_test_local_after
temp_workers_accuracy[idx] = acc_test_local_after
temp_workers_loss_test[idx] = loss_test_local_after
temp_workers_loss_differenc[idx] = loss_test_local_before - loss_test_local_after
temp_workers_accuracy_differenc[idx] = acc_test_local_after - acc_test_local_before
print("train finished")
while len(w_locals) < 1:
#print("recieving started")
index = 0
for idx in list_of_random_workers_newfl:
#print("acc is ", temp_workers_accuracy[idx])
# print(temp_workers_loss_differenc)
if workers_count >= m:
break
elif temp_workers_loss_differenc[idx] >= (threshold) \
and temp_workers_loss_differenc[idx] > 0 \
and flag[idx]==0:
print("Update Received")
w_locals.append(copy.deepcopy(temp_w_locals[index]))
#print(temp_w_locals[index])
loss_locals.append(temp_workers_loss[idx])
flag[idx] = 1
workers_count += 1
workers_participation[idx][iter] = 1
data_DCFL['round_{}'.format(iter)].append({
'C': args.frac,
'User ID': idx,
'Loss Train': loss_workers_total[idx, iter],
'Loss Test': temp_workers_loss[idx],
'Accuracy': temp_workers_accuracy[idx]
})
index += 1
if len(w_locals) < 1:
threshold = threshold / 2
if threshold == -np.inf:
threshold = 1
print("threshold increased to ", threshold)
# update global weights
w_glob = FedAvg(w_locals)
# copy weight to net_glob
net_glob.load_state_dict(w_glob)
#print("round completed")
loss_avg = sum(loss_locals) / len(loss_locals)
loss_train.append(loss_avg)
workers_percent_dist.append(workers_count/args.num_users)
counter_threshold_decrease[iter] = counter_threshold
print(iter, " round dist fl finished")
acc_test_final, loss_test_final = test_img(net_glob, dataset_test, args)
while_counter = loss_test_final
data_Global_DCFL["Round"].append(iter)
data_Global_DCFL["C"].append(args.frac)
data_Global_DCFL["Average Loss Train"].append(loss_avg)
data_Global_DCFL["Accuracy Test"].append(Global_Accuracy_Tracker[iter])
data_Global_DCFL["Average Loss Test"].append(Global_Loss_Tracker[iter])
data_Global_DCFL["Workers Number"].append(workers_count)
data_Global_DCFL["Large Test Loss"].append(float(loss_test_final))
data_Global_DCFL["Large Test Accuracy"].append(float(acc_test_final))
total_rounds_dcfl = iter
iter = iter + 1
#plot workers percent of participating
workers_percent_final = np.zeros(args.num_users)
workers_name = np.empty(args.num_users)
#print(workers_participation)
for i in range(len(workers_participation[:, 1])):
workers_percent_final[i] = sum(workers_participation[i, :])/args.epochs
workers_name[i] = i
workers_fraction_each_round_newFL = sum(workers_percent_final)/len(workers_percent_final)
# testing
#print("testing started")
net_glob.eval()
#print("train test started")
acc_train_final, loss_train_final = test_img(net_glob, dataset_train, args)
#print("train test finished")
acc_test_final, loss_test_final = test_img(net_glob, dataset_test, args)
acc_full_distributed.append(acc_test_final)
loss_full_ditributed.append(loss_test_final)
Final_LargeDataSetTest_DCFL["C"].append(args.frac)
Final_LargeDataSetTest_DCFL["Test Loss"].append(float(loss_test_final))
Final_LargeDataSetTest_DCFL["Test Accuracy"].append(float(acc_test_final))
Final_LargeDataSetTest_DCFL["Train Loss"].append(float(loss_train_final))
Final_LargeDataSetTest_DCFL["Train Accuracy"].append(float(acc_train_final))
Final_LargeDataSetTest_DCFL["Total Rounds"].append(int(total_rounds_dcfl))
variable_start = variable_start + while_counter
print("C is ", c_counter/10)
with open('CIFAR_100users_data_main_1229-2020.json', 'w') as outfile:
json.dump(data_main, outfile)
with open('CIFAR_100users_data_DCFL_1229-2020.json', 'w') as outfile:
json.dump(data_DCFL, outfile)
with open('CIFAR_100users_data_DCFL_Global_1229-2020.json', 'w') as outfile:
json.dump(data_Global_DCFL, outfile)
with open('CIFAR_100users_data_main_Global_1229-2020.json', 'w') as outfile:
json.dump(data_Global_main, outfile)
with open('Final-CIFAR_100users_data_main_Global_1229-2020.json', 'w') as outfile:
json.dump(Final_LargeDataSetTest_MainFL, outfile)
with open('Final-CIFAR_100users_data_DCFL_Global_1229-2020.json', 'w') as outfile:
json.dump(Final_LargeDataSetTest_DCFL, outfile)
return 1
dataset_test = datasets.CIFAR10('./data/cifar',
train=False,
download=True,
transform=trans_cifar)
if args.iid:
dict_users = cifar_iid(dataset_train, args.num_users, args.seed)
else:
exit('Error: only consider IID setting in CIFAR10')
else:
exit('Error: unrecognized dataset')
img_size = dataset_train[0][0].shape
# build model
if args.model == 'cnn' and args.dataset == 'cifar':
net_local = CNNCifar(args=args).to(args.device)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_local = CNNMnist(args=args).to(args.device)
elif args.model == 'mlp':
len_in = 1
for x in img_size:
len_in *= x
net_local = MLP(dim_in=len_in, dim_hidden=64,
dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
#Let's train the model few epochs first
n_epochs = 3
batch_size_train = 64
sys.exit(0)
if args.manual_seed:
init_random_seed(args.manual_seed)
# load dataset and split users
dataset_train, dict_users, dataset_test, test_dict_users, p_k, img_size, users_classes = data_split(args)
########################## centralized learning #############################
if args.centralized and (not os.path.exists(filename_nn)):
print('centralized learning...')
# build model for centralized learning
if (args.model == 'cnn' ) and args.dataset == 'mnist':
net_glob = CNN(args=args).to(args.device)
elif (args.model == 'cnn') and args.dataset == 'cifar':
net_glob = CNNCifar(args=args).to(args.device)
elif args.model == 'mlp':
len_in = 1
for x in img_size:
len_in *= x
net_glob = MLP(dim_in=len_in, dim_hidden=200,
dim_out=args.num_classes).to(args.device)
elif args.model == 'softmax':
len_in = 1
for x in img_size:
len_in *= x
net_glob = SoftmaxClassifier(dim_in=len_in,
dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
# For DH protocol
shared_p, shared_g, cnt_comm = 6362166871434581, 13, 0
# simulate sending message
url = 'http://10.28.156.99:6789'
# t-out-of-n poly
poly = [random.randint(3, shared_p) for _ in range(args.num_users - 1)]
print('DemoFL' if _Mode == 0 else 'FedAvg')
args.device = torch.device('cuda:{}'.format(
args.gpu) if torch.cuda.is_available() and args.gpu != -1 else 'cpu')
##### init global model
if args.dataset == 'cifar':
args.num_channels = 3
args.iid = True
net_global = CNNCifar(args=args).to(args.device)
else:
args.num_channels = 1
args.iid = False
net_global = CNNMnist(args=args).to(args.device)
net_global.train()
data_train, data_test, dict_users = load_data(args.dataset, args.iid,
args.num_users)
##### copy w structure, create zero base
w_glob = net_global.state_dict()
w_zero = copy.deepcopy(w_glob)
for i in w_zero:
w_zero[i] *= 0.
dataset_test = datasets.CIFAR10('./data/cifar',
train=False,
download=True,
transform=trans_cifar)
if args.iid:
dict_users = cifar_iid(dataset_train, args.num_users, args.seed)
else:
exit('Error: only consider IID setting in CIFAR10')
else:
exit('Error: unrecognized dataset')
img_size = dataset_train[0][0].shape
# build model
if args.model == 'cnn' and args.dataset == 'cifar':
net_local = CNNCifar(args=args).to(args.device)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_local = CNNMnist(args=args).to(args.device)
elif args.model == 'mlp':
len_in = 1
for x in img_size:
len_in *= x
net_local = MLP(dim_in=len_in, dim_hidden=64,
dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
if args.new:
print('======created a new model========:\n', net_local)
else:
checkpoint = torch.load(args.base_file)
def main():
manualSeed = 1
np.random.seed(manualSeed)
random.seed(manualSeed)
torch.manual_seed(manualSeed)
# if you are suing GPU
torch.cuda.manual_seed(manualSeed)
torch.cuda.manual_seed_all(manualSeed)
torch.backends.cudnn.enabled = False
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
# parse args
args = args_parser()
args.device = torch.device('cuda:{}'.format(
args.gpu) if torch.cuda.is_available() and args.gpu != -1 else 'cpu')
# load dataset and split users
if args.dataset == 'mnist':
trans_mnist = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])
dataset_train = datasets.MNIST('../data/mnist/',
train=True,
download=True,
transform=trans_mnist)
dataset_test = datasets.MNIST('../data/mnist/',
train=False,
download=True,
transform=trans_mnist)
# sample users
if args.iid:
dict_users_DCFL = mnist_iid(dataset_train, args.num_users)
else:
dict_users_DCFL, dict_labels_counter = mnist_noniid(
dataset_train, args.num_users)
dict_users_mainFL, dict_labels_counter_mainFL = dict_users_DCFL, dict_labels_counter
elif args.dataset == 'cifar':
trans_cifar = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
dataset_train = datasets.CIFAR10('../data/cifar',
train=True,
download=True,
transform=trans_cifar)
dataset_test = datasets.CIFAR10('../data/cifar',
train=False,
download=True,
transform=trans_cifar)
if args.iid:
dict_users_DCFL = cifar_iid(dataset_train, args.num_users)
dict_users_mainFL = dict_users_DCFL
dict_labels_counter_mainFL = dict()
dict_labels_counter = dict()
else:
dict_users_DCFL, dict_labels_counter = cifar_noniid(
dataset_train, args.num_users)
dict_users_mainFL, dict_labels_counter_mainFL = dict_users_DCFL, dict_labels_counter
elif args.dataset == 'fmnist':
trans_fmnist = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])
dataset_train = datasets.FashionMNIST('../data/fmnist',
train=True,
download=True,
transform=trans_fmnist)
dataset_test = datasets.FashionMNIST('../data/fmnist',
train=False,
download=True,
transform=trans_fmnist)
if args.iid:
print("iid")
dict_users_DCFL = mnist_iid(dataset_train, args.num_users)
else:
print("non iid")
dict_users_DCFL, dict_labels_counter = mnist_noniid(
dataset_train, args.num_users)
dict_users_mainFL, dict_labels_counter_mainFL = dict_users_DCFL, dict_labels_counter
else:
exit('Error: unrecognized dataset')
img_size = dataset_train[0][0].shape
# Small shared dataset
test_ds, valid_ds_before = torch.utils.data.random_split(
dataset_test, (9500, 500))
small_shared_dataset = create_shared_dataset(valid_ds_before, 200)
optimal_delay = 1.0
# Start process for each fraction of c
for c_counter in range(3, 3 + 1, 2):
if args.model == 'cnn' and args.dataset == 'cifar':
net_glob = CNNCifar(args=args).to(args.device)
# net_glob_mainFL = copy.deepcopy(net_glob)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_glob = CNNMnist(args=args).to(args.device)
# net_glob_mainFL = copy.deepcopy(net_glob)
elif args.model == 'cnn' and args.dataset == 'fmnist':
net_glob = CNNFashion_Mnist(args=args).to(args.device)
# net_glob_mainFL = copy.deepcopy(net_glob)
elif args.model == 'mlp':
len_in = 1
for x in img_size:
len_in *= x
net_glob = MLP(dim_in=len_in,
dim_hidden=200,
dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
# Saving data
data_Global_main = {
"C": [],
"Round": [],
"Average Loss Train": [],
"SDS Loss": [],
"SDS Accuracy": [],
"Workers Number": [],
"Large Test Loss": [],
"Large Test Accuracy": [],
"Communication Cost": []
}
Final_LargeDataSetTest_MainFL = {
"C": [],
"Test Accuracy": [],
"Test Loss": [],
"Train Loss": [],
"Train Accuracy": [],
"Total Rounds": [],
"Communication Cost": []
}
data_Global_DCFL = {
"C": [],
"Round": [],
"Average Loss Train": [],
"SDS Loss": [],
"SDS Accuracy": [],
"Workers Number": [],
"Large Test Loss": [],
"Large Test Accuracy": [],
"Communication Cost": []
}
Final_LargeDataSetTest_DCFL = {
"C": [],
"Test Accuracy": [],
"Test Loss": [],
"Train Loss": [],
"Train Accuracy": [],
"Total Rounds": [],
"Communication Cost": []
}
data_Global_G1 = {
"C": [],
"Round": [],
"Average Loss Train": [],
"SDS Loss": [],
"SDS Accuracy": [],
"Workers Number": [],
"Large Test Loss": [],
"Large Test Accuracy": [],
"Communication Cost": []
}
Final_LargeDataSetTest_G1 = {
"C": [],
"Test Accuracy": [],
"Test Loss": [],
"Train Loss": [],
"Train Accuracy": [],
"Total Rounds": [],
"Communication Cost": []
}
data_Global_G2 = {
"C": [],
"Round": [],
"Average Loss Train": [],
"SDS Loss": [],
"SDS Accuracy": [],
"Workers Number": [],
"Large Test Loss": [],
"Large Test Accuracy": [],
"Communication Cost": []
}
Final_LargeDataSetTest_G2 = {
"C": [],
"Test Accuracy": [],
"Test Loss": [],
"Train Loss": [],
"Train Accuracy": [],
"Total Rounds": [],
"Communication Cost": []
}
data_Global_Muhammed = {
"C": [],
"Round": [],
"Average Loss Train": [],
"SDS Loss": [],
"SDS Accuracy": [],
"Workers Number": [],
"Large Test Loss": [],
"Large Test Accuracy": [],
"Communication Cost": []
}
Final_LargeDataSetTest_Muhammed = {
"C": [],
"Test Accuracy": [],
"Test Loss": [],
"Train Loss": [],
"Train Accuracy": [],
"Total Rounds": [],
"Communication Cost": []
}
data_Global_Cho = {
"C": [],
"Round": [],
"Average Loss Train": [],
"SDS Loss": [],
"SDS Accuracy": [],
"Workers Number": [],
"Large Test Loss": [],
"Large Test Accuracy": [],
"Communication Cost": []
}
Final_LargeDataSetTest_Cho = {
"C": [],
"Test Accuracy": [],
"Test Loss": [],
"Train Loss": [],
"Train Accuracy": [],
"Total Rounds": [],
"Communication Cost": []
}
net_glob.train()
net_glob_mainFL = copy.deepcopy(net_glob)
net_glob_G1 = copy.deepcopy(net_glob)
net_glob_G2 = copy.deepcopy(net_glob)
cost = np.random.rand(args.num_users)
R_G1 = 5
args.frac = (c_counter / 10)
# Main FL
loss_main, dict_workers_index, Final_LargeDataSetTest_MainFL_temp, data_Global_main_temp = mainFl(
net_glob_mainFL, dict_users_mainFL, dict_labels_counter_mainFL,
args, cost, dataset_train, dataset_test, small_shared_dataset)
Final_LargeDataSetTest_MainFL = merge(
Final_LargeDataSetTest_MainFL, Final_LargeDataSetTest_MainFL_temp)
data_Global_main = merge(data_Global_main, data_Global_main_temp)
# with open(os.path.join(OUT_DIR, f"dict_users_mainFL-C-{args.frac}-{args.dataset}.pkl"), 'wb') as file:
# pickle.dump(dict_users_mainFL, file)
# with open(os.path.join(OUT_DIR, f"dict_users_mainFL-C-{args.frac}-{args.dataset}.pkl"), 'rb') as file:
# dict_users_mainFL = pickle.load(file)
# with open(os.path.join(OUT_DIR, f"workers_index-C-{args.frac}-{args.dataset}.pkl"), 'wb') as file:
# pickle.dump(dict_workers_index, file)
# with open(os.path.join(OUT_DIR, f"cost-C-{args.frac}-{args.dataset}.pkl"), 'wb') as file:
# pickle.dump(cost, file)
# with open(os.path.join(OUT_DIR, f"cost-C-{args.frac}-{args.dataset}.pkl"), 'rb') as file:
# cost = pickle.load(file)
# print(cost)
# with open(os.path.join(OUT_DIR, f"GoalLoss-C-{args.frac}-{args.dataset}.pkl"), 'wb') as file:
# pickle.dump(loss_main, file)
date = datetime.now()
_dir = os.path.join(OUT_DIR, str(date.date()))
if not os.path.exists(_dir):
os.makedirs(_dir)
save_time = time.strftime("%Y%m%d-%H%M%S")
Final_LargeDataSetTest_MainFL = pd.DataFrame.from_dict(
Final_LargeDataSetTest_MainFL)
data_Global_main = pd.DataFrame.from_dict(data_Global_main)
Final_LargeDataSetTest_MainFL.to_csv(
os.path.join(
_dir,
f"{save_time}-{args.dataset}-Final_LargeDataSetTest_MainFL.csv"
))
data_Global_main.to_csv(
os.path.join(_dir,
f"{save_time}-{args.dataset}-data_Global_main.csv"))
# Proposed G1
Final_LargeDataSetTest_G1_temp, data_Global_G1_temp = Proposed_G1(
net_glob_G1, dict_workers_index, dict_users_DCFL,
dict_labels_counter_mainFL, args, cost, dataset_train,
dataset_test, small_shared_dataset, loss_main, R_G1, optimal_delay)
Final_LargeDataSetTest_G1 = merge(Final_LargeDataSetTest_G1,
Final_LargeDataSetTest_G1_temp)
data_Global_G1 = merge(data_Global_G1, data_Global_G1_temp)
Final_LargeDataSetTest_G1 = pd.DataFrame.from_dict(
Final_LargeDataSetTest_G1)
data_Global_G1 = pd.DataFrame.from_dict(data_Global_G1)
Final_LargeDataSetTest_G1.to_csv(
os.path.join(
_dir,
f"{save_time}-{args.dataset}-Final_LargeDataSetTest_G1.csv"))
data_Global_G1.to_csv(
os.path.join(_dir,
f"{save_time}-{args.dataset}-data_Global_G1.csv"))
print("G1 alg is done")
dataset_test = datasets.CIFAR10('../data/cifar',
train=False,
download=True,
transform=trans_cifar)
if args.iid:
dict_users = cifar_iid(dataset_train, args.num_users)
else:
exit('Error: only consider IID setting in CIFAR10')
else:
exit('Error: unrecognized dataset')
img_size = dataset_train[0][0].shape
# build model
if args.model == 'cnn' and args.dataset == 'cifar':
global_net = CNNCifar(args=args).to(args.device)
elif args.model == 'cnn' and args.dataset == 'mnist':
global_net = CNNMnist(args=args).to(args.device)
elif args.model == 'mlp':
len_in = 1
for x in img_size:
len_in *= x
global_net = MLP(dim_in=len_in,
dim_hidden=200,
dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
print(global_net)
global_net.train()
dataset_test = datasets.CIFAR10('./data/cifar',
train=False,
download=True,
transform=trans_cifar)
if args.iid:
dict_users = cifar_iid(dataset_train,
args.num_users) #utils/sampling.py
else:
exit('Error: only consider IID setting in CIFAR10')
else:
exit('Error: unrecognized dataset')
img_size = dataset_train[0][0].shape
# build model
if args.model == 'cnn' and args.dataset == 'cifar':
net_glob = CNNCifar(args=args).to(args.device)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_glob = CNNMnist(args=args).to(args.device)
elif args.model == 'mlp':
len_in = 1
for x in img_size:
len_in *= x
net_glob = MLP(dim_in=len_in, dim_hidden=200,
dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
print(net_glob)
#FedAvg 代码的核心；每个迭代轮次本地更新 --> 复制参与本轮更新的 users 的所有权重 w_locals --> 通过定义的 FedAvg 函数求模型参数的平均 --> 分发到每个用户进行更新
# 在server端，初始化模型参数w_glob，并将原始的模型参数广播给所有的clients
net_glob.train()
exit('Error: unrecognized dataset')
a = np.arange(len(dataset_test))
np.random.shuffle(a)
dataset_valid = DatasetSplit(dataset=dataset_test, idxs=a[:1000])
dataset_test = DatasetSplit(dataset=dataset_test, idxs=a[1000:])
if args.iid == 'noniid_ssl' and args.dataset == 'cifar':
dict_users, dict_users_labeled, pseudo_label = noniid_ssl(
dataset_train_weak, args.num_users, args.label_rate)
else:
dict_users, dict_users_labeled, pseudo_label = sample(
dataset_train_weak, args.num_users, args.label_rate, args.iid)
if args.dataset == 'cifar':
net_glob = CNNCifar(args=args).to(args.device)
net_glob_helper_1 = CNNCifar(args=args).to(args.device)
net_glob_helper_2 = CNNCifar(args=args).to(args.device)
net_glob_valid = CNNCifar(args=args).to(args.device)
elif args.dataset == 'mnist':
net_glob = CNNMnist(args=args).to(args.device)
net_glob_helper_1 = CNNMnist(args=args).to(args.device)
net_glob_helper_2 = CNNMnist(args=args).to(args.device)
net_glob_valid = CNNMnist(args=args).to(args.device)
elif args.dataset == 'svhn':
net_glob = CNNCifar(args=args).to(args.device)
net_glob_helper_1 = CNNCifar(args=args).to(args.device)
net_glob_helper_2 = CNNCifar(args=args).to(args.device)
net_glob_valid = CNNCifar(args=args).to(args.device)
test_sum += test_count_dict[items]
print("TEST SUM", test_sum)
###ANALYZING END
###
if args.iid:
dict_users = cifar_iid(dataset_train, args.num_users)
else:
exit('Error: only consider IID setting in CIFAR10')
else:
exit('Error: unrecognized dataset')
img_size = dataset_train[0][0].shape
# build model
if args.model == 'cnn' and args.dataset == 'cifar':
net_glob = CNNCifar(args=args).to(args.device)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_glob = CNNMnist(args=args).to(args.device)
net_glob1 = CNNMnist(args=args).to(args.device)
net_glob5 = CNNMnist(args=args).to(args.device)
net_glob10 = CNNMnist(args=args).to(args.device)
net_glob15 = CNNMnist(args=args).to(args.device)
net_glob20 = CNNMnist(args=args).to(args.device)
net_glob25 = CNNMnist(args=args).to(args.device)
net_glob30 = CNNMnist(args=args).to(args.device)
elif args.model == 'mlp':
len_in = 1
for x in img_size:
len_in *= x
net_glob = MLP(dim_in=len_in, dim_hidden=64,
dim_out=args.num_classes).to(args.device)
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
]))
img_size = dataset_train[0][0].shape
elif args.dataset == 'cifar':
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
dataset_train = datasets.CIFAR10('./data/cifar', train=True, transform=transform, target_transform=None, download=True)
img_size = dataset_train[0][0].shape
else:
exit('Error: unrecognized dataset')
# build model
if args.model == 'cnn' and args.dataset == 'cifar':
net_glob = CNNCifar(args=args).to(args.device)
elif args.model == 'cnn' and args.dataset == 'mnist':
net_glob = CNNMnist(args=args).to(args.device)
elif args.model == 'mlp':
len_in = 1
for x in img_size:
len_in *= x
net_glob = MLP(dim_in=len_in, dim_hidden=64, dim_out=args.num_classes).to(args.device)
else:
exit('Error: unrecognized model')
print(net_glob)
# training
optimizer = optim.SGD(net_glob.parameters(), lr=args.lr, momentum=args.momentum)
train_loader = DataLoader(dataset_train, batch_size=64, shuffle=True)