test_loader = DataLoader(dataset_test, batch_size=1000, shuffle=False) img_size = dataset_train[0][0].shape # build model if args.model == 'lenet' and (args.dataset == 'cifar' or args.dataset == 'fmnist'): net_glob = CNNCifar(args=args).to(args.device) elif args.model == 'vgg' and args.dataset == 'cifar': net_glob = vgg16().to(args.device) else: exit('Error: unrecognized model') print(net_glob) net_glob.train() # copy weights w_init = copy.deepcopy(net_glob.state_dict()) local_acc_final = [] total_acc_final = [] local_acc = np.zeros([args.num_users, args.epochs]) total_acc = np.zeros([args.num_users, args.epochs]) # training for idx in range(args.num_users): # print(w_init) net_glob.load_state_dict(w_init) optimizer = optim.Adam(net_glob.parameters()) train_loader = DataLoader(DatasetSplit(dataset_train, dict_users[idx]), batch_size=64, shuffle=True) image_trainset_weight = np.zeros(10)
net_glob = MLP(dim_in=len_in, dim_hidden=64, dim_out=args.num_classes).to(args.device) else: exit('Error: unrecognized model') print(net_glob) net_glob.train() net_glob1.train() net_glob5.train() net_glob10.train() net_glob15.train() net_glob20.train() net_glob25.train() net_glob30.train() # copy weights w_glob = net_glob.state_dict() w_glob1 = net_glob1.state_dict() w_glob5 = net_glob5.state_dict() w_glob10 = net_glob10.state_dict() w_glob15 = net_glob15.state_dict() w_glob20 = net_glob20.state_dict() w_glob25 = net_glob25.state_dict() w_glob30 = net_glob30.state_dict() # training - NO ATTACK loss_train = [] cv_loss, cv_acc = [], [] val_loss_pre, counter = 0, 0 net_best = None best_loss = None val_acc_list, net_list = [], []
def main(): # parse args args = args_parser() args.device = torch.device('cuda:{}'.format( args.gpu) if torch.cuda.is_available() and args.gpu != -1 else 'cpu') # load dataset and split users if args.dataset == 'mnist': trans_mnist = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ]) dataset_train = datasets.MNIST('../data/mnist/', train=True, download=True, transform=trans_mnist) dataset_test = datasets.MNIST('../data/mnist/', train=False, download=True, transform=trans_mnist) print("type of test dataset", type(dataset_test)) # sample users if args.iid: dict_users = mnist_iid(dataset_train, args.num_users) else: dict_users, dict_labels_counter = mnist_noniid( dataset_train, args.num_users) dict_users_2, dict_labels_counter_2 = dict_users, dict_labels_counter #dict_users, dict_labels_counter = mnist_noniid_unequal(dataset_train, args.num_users) elif args.dataset == 'cifar': trans_cifar = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ]) dataset_train = datasets.CIFAR10('../data/cifar', train=True, download=True, transform=trans_cifar) dataset_test = datasets.CIFAR10('../data/cifar', train=False, download=True, transform=trans_cifar) if args.iid: dict_users = cifar_iid(dataset_train, args.num_users) else: exit('Error: only consider IID setting in CIFAR10') else: exit('Error: unrecognized dataset') img_size = dataset_train[0][0].shape # build model if args.model == 'cnn' and args.dataset == 'cifar': net_glob = CNNCifar(args=args).to(args.device) net_glob_2 = CNNCifar(args=args).to(args.device) elif args.model == 'cnn' and args.dataset == 'mnist': net_glob = CNNMnist(args=args).to(args.device) net_glob_2 = CNNMnist(args=args).to(args.device) elif args.model == 'mlp': len_in = 1 for x in img_size: len_in *= x net_glob = MLP(dim_in=len_in, dim_hidden=200, dim_out=args.num_classes).to(args.device) else: exit('Error: unrecognized model') #print(net_glob) #net_glob.train() acc_test, loss_test = test_img(net_glob, dataset_test, args) print("val test finished") print("{:.2f}".format(acc_test)) temp = net_glob #net_glob_2 = net_glob temp_2 = net_glob_2 # copy weights w_glob = net_glob.state_dict() # training loss_train = [] cv_loss, cv_acc = [], [] val_loss_pre, counter = 0, 0 net_best = None best_loss = None val_acc_list, net_list = [], [] Loss_local_each_global_total = [] test_ds, valid_ds = torch.utils.data.random_split(dataset_test, (9500, 500)) loss_workers_total = np.zeros(shape=(args.num_users, args.epochs)) label_workers = { i: np.array([], dtype='int64') for i in range(args.num_users) } workers_percent = [] workers_count = 0 acc_test_global, loss_test_global = test_img(x, valid_ds, args) selected_users_index = [] for idx in range(args.num_users): # print("train started") local = LocalUpdate(args=args, dataset=dataset_train, idxs=dict_users[idx]) w, loss = local.train(net=copy.deepcopy(net_glob).to(args.device)) # print(w) # print("train completed") # temp = FedAvg(w) temp.load_state_dict(w) temp.eval() acc_test_local, loss_test_local = test_img(temp, valid_ds, args) loss_workers_total[idx, iter] = acc_test_local if workers_count >= (args.num_users / 2): break elif acc_test_local >= (0.7 * acc_test_global): selected_users_index.append(idx) for iter in range(args.epochs): print("round started") Loss_local_each_global = [] loss_workers = np.zeros((args.num_users, args.epochs)) w_locals, loss_locals = [], [] m = max(int(args.frac * args.num_users), 1) #idxs_users = np.random.choice(range(args.num_users), m, replace=False) #if iter % 5 == 0: # Minoo x = net_glob x.eval() Loss_local_each_global_total.append(acc_test_global) for idx in selected_users_index: #print("train started") local = LocalUpdate(args=args, dataset=dataset_train, idxs=dict_users[idx]) w, loss = local.train(net=copy.deepcopy(net_glob).to(args.device)) #print(w) #print("train completed") #temp = FedAvg(w) temp.load_state_dict(w) temp.eval() acc_test_local, loss_test_local = test_img(temp, valid_ds, args) loss_workers_total[idx, iter] = acc_test_local if workers_count >= (args.num_users / 2): break elif acc_test_local >= (0.7 * acc_test_global): w_locals.append(copy.deepcopy(w)) loss_locals.append(copy.deepcopy(loss)) print("Update Received") workers_count += 1 # update global weights w_glob = FedAvg(w_locals) # copy weight to net_glob net_glob.load_state_dict(w_glob) print("round completed") loss_avg = sum(loss_locals) / len(loss_locals) print('Round {:3d}, Average loss {:.3f}'.format(iter, loss_avg)) loss_train.append(loss_avg) workers_percent.append(workers_count) # plot loss curve plt.figure() plt.plot(range(len(workers_percent)), workers_percent) plt.ylabel('train_loss') plt.savefig( './save/Newfed_WorkersPercent_0916_{}_{}_{}_C{}_iid{}.png'.format( args.dataset, args.model, args.epochs, args.frac, args.iid)) # print(loss_workers_total) # plot loss curve # plt.figure() # plt.plot(range(len(loss_train)), loss_train) # plt.ylabel('train_loss') # plt.savefig('./save/Newfed_0916_{}_{}_{}_C{}_iid{}.png'.format(args.dataset, args.model, args.epochs, args.frac, args.iid)) # plt.figure() for i in range(args.num_users): plot = plt.plot(range(len(loss_workers_total[i, :])), loss_workers_total[i, :], label="Worker {}".format(i)) plot5 = plt.plot(range(len(Loss_local_each_global_total)), Loss_local_each_global_total, color='000000', label="Global") plt.legend(loc='best') plt.ylabel('Small Test Set Accuracy of workers') plt.xlabel('Number of Rounds') plt.savefig( './save/NewFed_2workers_Acc_0916_{}_{}_{}_C{}_iid{}.png'.format( args.dataset, args.model, args.epochs, args.frac, args.iid)) # plt.figure() # bins = np.linspace(0, 9, 3) # a = dict_labels_counter[:, 0].ravel() # print(type(a)) # b = dict_labels_counter[:, 1].ravel() # x_labels = ['0', '1', '2', '3','4','5','6','7','8','9'] # # Set plot parameters # fig, ax = plt.subplots() # width = 0.1 # width of bar # x = np.arange(10) # ax.bar(x, dict_labels_counter[:, 0], width, color='#000080', label='Worker 1') # ax.bar(x + width, dict_labels_counter[:, 1], width, color='#73C2FB', label='Worker 2') # ax.bar(x + 2*width, dict_labels_counter[:, 2], width, color='#ff0000', label='Worker 3') # ax.bar(x + 3*width, dict_labels_counter[:, 3], width, color='#32CD32', label='Worker 4') # ax.set_ylabel('Number of Labels') # ax.set_xticks(x + width + width / 2) # ax.set_xticklabels(x_labels) # ax.set_xlabel('Labels') # ax.legend() # plt.grid(True, 'major', 'y', ls='--', lw=.5, c='k', alpha=.3) # fig.tight_layout() # plt.savefig( # './save/Newfed_2workersLabels_0916_{}_{}_{}_C{}_iid{}.png'.format(args.dataset, args.model, args.epochs, args.frac, # args.iid)) # testing print("testing started") net_glob.eval() print("train test started") acc_train_final, loss_train_final = test_img(net_glob, dataset_train, args) print("train test finished") acc_test_final, loss_test_final = test_img(net_glob, dataset_test, args) print("val test finished") #print("Training accuracy: {:.2f}".format(acc_train)) #print("Testing accuracy: {:.2f}".format(acc_test)) print("{:.2f}".format(acc_test_final)) #print("{:.2f".format(Loss_local_each_worker)) # training w_glob_2 = net_glob_2.state_dict() loss_train_2 = [] cv_loss_2, cv_acc_2 = [], [] val_loss_pre_2, counter_2 = 0, 0 net_best_2 = None best_loss_2 = None val_acc_list_2, net_list_2 = [], [] Loss_local_each_global_total_2 = [] loss_workers_total_2 = np.zeros(shape=(args.num_users, args.epochs)) label_workers_2 = { i: np.array([], dtype='int64') for i in range(args.num_users) } for iter in range(args.epochs): print("round started") Loss_local_each_global_2 = [] loss_workers_2 = np.zeros((args.num_users, args.epochs)) w_locals_2, loss_locals_2 = [], [] m_2 = max(int(args.frac * args.num_users), 1) idxs_users_2 = np.random.choice(range(args.num_users), m_2, replace=False) # Minoo x_2 = net_glob_2 x_2.eval() acc_test_global_2, loss_test_global_2 = test_img(x_2, valid_ds, args) Loss_local_each_global_total_2.append(acc_test_global_2) for idx in idxs_users_2: #print("train started") local_2 = LocalUpdate(args=args, dataset=dataset_train, idxs=dict_users_2[idx]) w_2, loss_2 = local_2.train( net=copy.deepcopy(net_glob_2).to(args.device)) #print(w) #print("train completed") w_locals_2.append(copy.deepcopy(w_2)) loss_locals_2.append(copy.deepcopy(loss_2)) #temp = FedAvg(w) temp_2.load_state_dict(w_2) temp_2.eval() acc_test_local_2, loss_test_local_2 = test_img( temp_2, valid_ds, args) loss_workers_total_2[idx, iter] = acc_test_local_2 # update global weights w_glob_2 = FedAvg(w_locals_2) # copy weight to net_glob net_glob_2.load_state_dict(w_glob_2) print("round completed") loss_avg_2 = sum(loss_locals_2) / len(loss_locals_2) print('Round {:3d}, Average loss {:.3f}'.format(iter, loss_avg_2)) loss_train_2.append(loss_avg_2) print("round completed") # plot loss curve plt.figure() plt.plot(range(len(loss_train_2)), loss_train_2, color='#000000', label="Main FL") plt.plot(range(len(loss_train)), loss_train, color='#ff0000', label="Centralized Algorithm") plt.ylabel('train_loss') plt.savefig('./save/main_fed_0916_{}_{}_{}_C{}_iid{}.png'.format( args.dataset, args.model, args.epochs, args.frac, args.iid)) # print(loss_workers_total) plt.figure() for i in range(args.num_users): plot = plt.plot(range(len(loss_workers_total_2[i, :])), loss_workers_total_2[i, :], label="Worker {}".format(i)) plot5 = plt.plot(range(len(Loss_local_each_global_total_2)), Loss_local_each_global_total_2, color='000000', label="Global") plt.legend(loc='best') plt.ylabel('Small Test Set Accuracy of workers') plt.xlabel('Number of Rounds') plt.savefig('./save/mainfed_Acc_0916_{}_{}_{}_C{}_iid{}.png'.format( args.dataset, args.model, args.epochs, args.frac, args.iid)) # plt.figure() # bins = np.linspace(0, 9, 3) # a = dict_labels_counter_2[:, 0].ravel() # print(type(a)) # b = dict_labels_counter_2[:, 1].ravel() # x_labels = ['0', '1', '2', '3','4','5','6','7','8','9'] # # Set plot parameters # fig, ax = plt.subplots() # width = 0.1 # width of bar # x = np.arange(10) # ax.bar(x, dict_labels_counter_2[:, 0], width, color='#000080', label='Worker 1') # ax.bar(x + width, dict_labels_counter_2[:, 1], width, color='#73C2FB', label='Worker 2') # ax.bar(x + 2*width, dict_labels_counter_2[:, 2], width, color='#ff0000', label='Worker 3') # ax.bar(x + 3*width, dict_labels_counter_2[:, 3], width, color='#32CD32', label='Worker 4') # ax.set_ylabel('Number of Labels') # ax.set_xticks(x + width + width / 2) # ax.set_xticklabels(x_labels) # ax.set_xlabel('Labels') # ax.legend() # plt.grid(True, 'major', 'y', ls='--', lw=.5, c='k', alpha=.3) # fig.tight_layout() # plt.savefig( # './save/main_fed_2workersLabels_0916_{}_{}_{}_C{}_iid{}.png'.format(args.dataset, args.model, args.epochs, args.frac, # args.iid)) # testing print("testing started") net_glob.eval() print("train test started") acc_train_final, loss_train_final = test_img(net_glob, dataset_train, args) print("train test finished") acc_test_final, loss_test_final = test_img(net_glob, dataset_test, args) print("val test finished") #print("Training accuracy: {:.2f}".format(acc_train)) #print("Testing accuracy: {:.2f}".format(acc_test)) print("{:.2f}".format(acc_test_final)) #print("{:.2f".format(Loss_local_each_worker)) return loss_test_final, loss_train_final
net_glob = CNNMnist(args=args).to(args.device) elif args.model == 'mlp': len_in = 1 for x in img_size: len_in *= x net_glob = MLP(dim_in=len_in, dim_hidden=200, dim_out=args.num_classes).to(args.device) else: exit('Error: unrecognized model') print(net_glob) #FedAvg 代码的核心;每个迭代轮次本地更新 --> 复制参与本轮更新的 users 的所有权重 w_locals --> 通过定义的 FedAvg 函数求模型参数的平均 --> 分发到每个用户进行更新 # 在server端,初始化模型参数w_glob,并将原始的模型参数广播给所有的clients net_glob.train() # copy weights w_glob = net_glob.state_dict() # 获取模型参数以共享,变量存放训练过程中需要学习的权重和偏执系数 # training loss_train = [] cv_loss, cv_acc = [], [] val_loss_pre, counter = 0, 0 net_best = None best_loss = None val_acc_list, net_list = [], [] if args.all_clients: print("Aggregation over all clients") w_locals = [w_glob for i in range(args.num_users)] for iter in range(args.epochs): # 每一全局模型更新轮次 loss_locals = [] if not args.all_clients:
##### init global model if args.dataset == 'cifar': args.num_channels = 3 args.iid = True net_global = CNNCifar(args=args).to(args.device) else: args.num_channels = 1 args.iid = False net_global = CNNMnist(args=args).to(args.device) net_global.train() data_train, data_test, dict_users = load_data(args.dataset, args.iid, args.num_users) ##### copy w structure, create zero base w_glob = net_global.state_dict() w_zero = copy.deepcopy(w_glob) for i in w_zero: w_zero[i] *= 0. ##### Clients clients = [FL_client(args) for _ in range(args.num_users)] for i, client in enumerate(clients): client.set_data(data_train, dict_users[i]) #********************************************************************************************** #********************************************************************************************** ##### Select Leaders leaders = np.random.choice(range(args.num_users), n_leader, replace=False) ##### Key exchange
elif args.dataset == 'mnist': net_glob = CNNMnist(args=args).to(args.device) net_ema_glob = CNNMnist(args=args).to(args.device) elif args.dataset == 'svhn': net_glob = CNNCifar(args=args).to(args.device) net_ema_glob = CNNCifar(args=args).to(args.device) else: exit('Error: unrecognized model') net_glob.train() net_ema_glob.train() # copy weights w_glob = net_glob.state_dict() w_ema_glob = net_ema_glob.state_dict() w_best = copy.deepcopy(w_glob) w_ema_best = copy.deepcopy(w_ema_glob) best_loss_valid = 100000000000 best_ema_loss_valid = 100000000000 # training loss_train = [] cv_loss, cv_acc = [], [] val_loss_pre, counter = 0, 0 net_best = None best_loss = None val_acc_list, net_list = [], [] dict_userepoch={i:0 for i in range(100)}
data, target = data.to(args.device), target.to(args.device) optimizer.zero_grad() output = net_glob(data) loss = creterion(output, target) loss.backward() optimizer.step() if batch_idx % 50 == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, batch_idx * len(data), len(train_loader.dataset), 100. * batch_idx / len(train_loader), loss.item())) batch_loss.append(loss.item()) # scheduler.step() loss_avg = sum(batch_loss)/len(batch_loss) print('\nTrain loss:', loss_avg) list_loss.append(loss_avg) writer.add_scalar('train_loss', loss_avg, epoch) test_acc, test_loss = test(args, net_glob, test_loader) writer.add_scalar('test_loss', test_loss, epoch) writer.add_scalar('test_acc', test_acc, epoch) # save model weights save_info = { "epochs": args.epochs, "optimizer": optimizer.state_dict(), "model": net_glob.state_dict() } save_path = f'save2/{TAG}' if args.debug else f'save2/{TAG}' torch.save(save_info, save_path) writer.close()
exit('Error: unrecognized dataset...') # split dataset {user_id: [list of data index]} dict_users_train, ratio = noniid_train(train_set, args.num_users) dict_users_test = noniid_test(test_set, args.num_users, ratio) print('Data finished...') # load global model net_glob = CNNCifar( args=args).to(args.device) if args.dataset == 'cifar' else CNNMnist( args=args).to(args.device) net_glob.train() # parameters w_glob = net_glob.state_dict() loss_train = [] # meta-learning for global initial parameters for epoch in range(args.meta_epochs): loss_locals = [] w_locals = [] m = max(int(args.frac * args.num_users), 1) idxs_users = np.random.choice(range(args.num_users), m, replace=False) for idx in idxs_users: client = Client(args=args, dataset=train_set, idxs=dict_users_train[idx], bs=args.train_bs) w, loss = client.reptile(
len_in *= x global_net = MLP(dim_in=len_in, dim_hidden=200, dim_out=args.num_classes).to(args.device) else: exit('Error: unrecognized model') print(global_net) global_net.train() # start time time_start = time.time() # copy weights w_glob = global_net.state_dict() loss_train = [] # training if args.all_clients: print("Aggregation over all clients") w_locals = [w_glob for i in range(args.num_users)] for round in range(args.epochs): loss_locals = [] if not args.all_clients: w_locals = [] # 设备上报状态特征 [xxxxxxxx]
# Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isdir('ckpt'), 'Error: no checkpoint directory found!' checkpoint = torch.load('./ckpt/vgg_full.pth') net_glob.load_state_dict(checkpoint['net']) net_best = checkpoint['acc'] start_iter = checkpoint['iter'] print('net_best: ', net_best) print('start_iter: ', start_iter) print(net_glob) nets_users = [] for i in range(args.num_users): nets_users.append([0, copy.deepcopy(net_glob.state_dict())]) # copy weights w_glob = net_glob.state_dict() # training loss_train = [] cv_loss, cv_acc = [], [] val_loss_pre, counter = 0, 0 best_loss = None val_acc_list, net_list = [], [] acc_train = [] loss_tr = [] acc_test = []
def main(): # parse args args = args_parser() args.device = torch.device('cuda:{}'.format(args.gpu) if torch.cuda.is_available() and args.gpu != -1 else 'cpu') # load dataset and split users if args.dataset == 'mnist': trans_mnist = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]) dataset_train = datasets.MNIST('../data/mnist/', train=True, download=True, transform=trans_mnist) dataset_test = datasets.MNIST('../data/mnist/', train=False, download=True, transform=trans_mnist) # sample users if args.iid: dict_users = mnist_iid(dataset_train, args.num_users) else: dict_users, dict_labels_counter = mnist_noniid(dataset_train, args.num_users) dict_users_mainFL, dict_labels_counter_mainFL = dict_users, dict_labels_counter elif args.dataset == 'cifar': trans_cifar = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) dataset_train = datasets.CIFAR10('../data/cifar', train=True, download=True, transform=trans_cifar) dataset_test = datasets.CIFAR10('../data/cifar', train=False, download=True, transform=trans_cifar) if args.iid: dict_users = cifar_iid(dataset_train, args.num_users) else: dict_users, dict_labels_counter = cifar_noniid(dataset_train, args.num_users) dict_users_mainFL, dict_labels_counter_mainFL = dict_users, dict_labels_counter elif args.dataset == 'fmnist': trans_fmnist = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]) dataset_train = datasets.FashionMNIST('../data/fmnist', train=True, download=True, transform=trans_fmnist) dataset_test = datasets.FashionMNIST('../data/fmnist', train=False, download=True, transform=trans_fmnist) if args.iid: dict_users = mnist_iid(dataset_train, args.num_users) else: dict_users, dict_labels_counter = mnist_noniid(dataset_train, args.num_users) dict_users_mainFL, dict_labels_counter_mainFL = dict_users, dict_labels_counter else: exit('Error: unrecognized dataset') img_size = dataset_train[0][0].shape acc_full_distributed = [] acc_full_main = [] loss_full_ditributed = [] loss_full_main = [] SD_acc_full_distributed = [] SD_acc_full_main = [] SD_loss_full_ditributed = [] SD_loss_full_main = [] workers_percent_full_distributed = [] workers_percent_full_main = [] variable_start = 0.1 variable_end = 1.0 while_counter = 0.1 counter_array = [] Accuracy_Fraction = [] Workers_Fraction = [] accuracy_fraction_each_round_newFL = 0 workers_fraction_each_round_newFL = 0 accuracy_fraction_each_round_mainFL = 0 workers_fraction_each_round_mainFL = 0 data_main = {} data_DCFL = {} data_Global_main = {"C": [], "Round":[], "Average Loss Train": [], "Average Loss Test": [], "Accuracy Test": [], "Workers Number": [], "Large Test Loss":[], "Large Test Accuracy":[]} data_Global_DCFL = {"C": [], "Round":[], "Average Loss Train": [], "Average Loss Test": [], "Accuracy Test": [], "Workers Number": [], "Large Test Loss":[], "Large Test Accuracy":[]} Final_LargeDataSetTest_DCFL = {"C":[], "Test Accuracy":[], "Test Loss":[], "Train Loss":[], "Train Accuracy":[], "Total Rounds":[]} Final_LargeDataSetTest_MainFL = {"C":[], "Test Accuracy": [], "Test Loss": [], "Train Loss": [], "Train Accuracy":[]} # build model args.frac = variable_start test_ds, valid_ds_before = torch.utils.data.random_split(dataset_test, (9500, 500)) valid_ds = create_shared_dataset(valid_ds_before, 200) #while variable_start <= variable_end: for c_counter in range(1, 11, 3): if args.model == 'cnn' and args.dataset == 'cifar': net_glob = CNNCifar(args=args).to(args.device) net_glob_mainFL = copy.deepcopy(net_glob) elif args.model == 'cnn' and args.dataset == 'mnist': net_glob = CNNMnist(args=args).to(args.device) net_glob_mainFL = copy.deepcopy(net_glob) elif args.model == 'cnn' and args.dataset == 'fmnist': net_glob = CNNFashion_Mnist(args=args).to(args.device) net_glob_mainFL = copy.deepcopy(net_glob) elif args.model == 'mlp': len_in = 1 for x in img_size: len_in *= x net_glob = MLP(dim_in=len_in, dim_hidden=200, dim_out=args.num_classes).to(args.device) else: exit('Error: unrecognized model') counter_array.append((c_counter/10)) args.frac = (c_counter/10) ######saving index of workers dict_workers_index = defaultdict(list) #############Main FL w_glob_mainFL = net_glob_mainFL.state_dict() loss_train_mainFL = [] # cv_loss_2, cv_acc_2 = [], [] # val_loss_pre_2, counter_2 = 0, 0 # net_best_2 = None # best_loss_2 = None # val_acc_list_2, net_list_2 = [], [] Loss_local_each_global_total_mainFL = [] Accuracy_local_each_global_total_mainFL = [] loss_workers_total_mainFL = np.zeros(shape=(args.num_users, args.epochs)) label_workers_mainFL = {i: np.array([], dtype='int64') for i in range(args.num_users)} validation_test_mainFed = [] acc_test, loss_test = test_img(net_glob_mainFL, dataset_test, args) workers_participation_main_fd = np.zeros((args.num_users, args.epochs)) workers_percent_main = [] # for iter in range(args.epochs): net_glob_mainFL.eval() acc_test_final_mainFL, loss_test_final_mainFL = test_img(net_glob_mainFL, dataset_test, args) while_counter_mainFL = loss_test_final_mainFL iter_mainFL = 0 workers_mainFL = [] for i in range(args.num_users): workers_mainFL.append(i) temp_netglob_mainFL = net_glob_mainFL while iter_mainFL < (args.epochs/2): data_main['round_{}'.format(iter_mainFL)] = [] # data_Global_main['round_{}'.format(iter)] = [] # print("round started") Loss_local_each_global_mainFL = [] loss_workers_mainFL = np.zeros((args.num_users, args.epochs)) w_locals_mainFL, loss_locals_mainFL = [], [] m_mainFL = max(int(args.frac * args.num_users), 1) idxs_users_mainFL = np.random.choice(range(args.num_users), m_mainFL, replace=False) list_of_random_workers = random.sample(workers_mainFL, m_mainFL) for i in range(len(list_of_random_workers)): dict_workers_index[iter_mainFL].append(list_of_random_workers[i]) x_mainFL = net_glob_mainFL x_mainFL.eval() acc_test_global_mainFL, loss_test_global_mainFL = test_img(x_mainFL, valid_ds, args) Loss_local_each_global_total_mainFL.append(loss_test_global_mainFL) Accuracy_local_each_global_total_mainFL.append(acc_test_global_mainFL) SD_acc_full_main.append(acc_test_global_mainFL) SD_loss_full_main.append(loss_test_global_mainFL) workers_count_mainFL = 0 temp_accuracy = np.zeros(1) temp_loss_test = np.zeros(1) temp_loss_train = np.zeros(1) for idx in list_of_random_workers: # print("train started") local_mainFL = LocalUpdate(args=args, dataset=dataset_train, idxs=dict_users_mainFL[idx]) w_mainFL, loss_mainFL = local_mainFL.train(net=copy.deepcopy(net_glob_mainFL).to(args.device)) # print(w) # print("train completed") w_locals_mainFL.append(copy.deepcopy(w_mainFL)) loss_locals_mainFL.append(copy.deepcopy(loss_mainFL)) # temp = FedAvg(w) temp_netglob_mainFL.load_state_dict(w_mainFL) temp_netglob_mainFL.eval() print(pnorm_2(temp_netglob_mainFL, 2)) acc_test_local_mainFL, loss_test_local_mainFL = test_img(temp_netglob_mainFL, valid_ds, args) temp_accuracy[0] = acc_test_local_mainFL temp_loss_test[0] = loss_test_local_mainFL temp_loss_train[0] = loss_mainFL loss_workers_total_mainFL[idx, iter_mainFL] = acc_test_local_mainFL workers_participation_main_fd[idx][iter_mainFL] = 1 workers_count_mainFL += 1 data_main['round_{}'.format(iter_mainFL)].append({ 'C': args.frac, 'User ID': idx, # 'Local Update': copy.deepcopy(w_mainFL), 'Loss Train': temp_loss_train[0], 'Loss Test': temp_loss_test[0], 'Accuracy': temp_accuracy[0] }) # update global weights w_glob_mainFL = FedAvg(w_locals_mainFL) # copy weight to net_glob net_glob_mainFL.load_state_dict(w_glob_mainFL) # print("round completed") loss_avg_mainFL = sum(loss_locals_mainFL) / len(loss_locals_mainFL) # print('Round {:3d}, Average loss {:.3f}'.format(iter, loss_avg_mainFL)) loss_train_mainFL.append(loss_avg_mainFL) # print("round completed") acc_test_round_mainfed, loss_test_round_mainfed = test_img(net_glob_mainFL, dataset_test, args) validation_test_mainFed.append(acc_test_round_mainfed) workers_percent_main.append(workers_count_mainFL / args.num_users) # plot workers percent of participating print(iter_mainFL, " round main fl finished") acc_test_final_mainFL, loss_test_final_mainFL = test_img(net_glob_mainFL, dataset_test, args) while_counter_mainFL = loss_test_final_mainFL data_Global_main["Round"].append(iter_mainFL) data_Global_main["C"].append(args.frac) data_Global_main["Average Loss Train"].append(float(loss_avg_mainFL)) data_Global_main["Average Loss Test"].append(float(loss_test_global_mainFL)) data_Global_main["Accuracy Test"].append(float(acc_test_global_mainFL)) data_Global_main["Workers Number"].append(float(workers_count_mainFL)) data_Global_main["Large Test Loss"].append(float(loss_test_final_mainFL)) data_Global_main["Large Test Accuracy"].append(float(acc_test_final_mainFL)) iter_mainFL = iter_mainFL + 1 workers_percent_final_mainFL = np.zeros(args.num_users) workers_name_mainFL = np.empty(args.num_users) for i in range(len(workers_participation_main_fd[:, 1])): workers_percent_final_mainFL[i] = sum(workers_participation_main_fd[i, :]) / args.epochs workers_name_mainFL[i] = i net_glob_mainFL.eval() # print("train test started") acc_train_final_main, loss_train_final_main = test_img(net_glob_mainFL, dataset_train, args) # print("train test finished") acc_test_final_main, loss_test_final_main = test_img(net_glob_mainFL, dataset_test, args) Final_LargeDataSetTest_MainFL["C"].append(args.frac) Final_LargeDataSetTest_MainFL["Test Loss"].append(float(loss_test_final_main)) Final_LargeDataSetTest_MainFL["Test Accuracy"].append(float(acc_test_final_main)) Final_LargeDataSetTest_MainFL["Train Loss"].append(float(loss_train_final_main)) Final_LargeDataSetTest_MainFL["Train Accuracy"].append(float(acc_train_final_main)) # copy weights w_glob = net_glob.state_dict() temp_after = copy.deepcopy(net_glob) temp_before = copy.deepcopy(net_glob) # training loss_train = [] # cv_loss, cv_acc = [], [] # val_loss_pre, counter = 0, 0 # net_best = None # best_loss = None # val_acc_list, net_list = [], [] Loss_local_each_global_total = [] # valid_ds = create_shared_dataset(dataset_test, 500) loss_workers_total = np.zeros(shape=(args.num_users, args.epochs)) label_workers = {i: np.array([], dtype='int64') for i in range(args.num_users)} workers_percent_dist = [] validation_test_newFed = [] workers_participation = np.zeros((args.num_users, args.epochs)) workers = [] for i in range(args.num_users): workers.append(i) counter_threshold_decrease = np.zeros(args.epochs) Global_Accuracy_Tracker = np.zeros(args.epochs) Global_Loss_Tracker = np.zeros(args.epochs) threshold = 0.5 alpha = 0.5 ##decrease parameter beta = 0.1 ##delta accuracy controller gamma = 0.5 ##threshold decrease parameter Goal_Loss = float(loss_test_final_main) #for iter in range(args.epochs): net_glob.eval() acc_test_final, loss_test_final = test_img(net_glob, dataset_test, args) while_counter = float(loss_test_final) iter = 0 total_rounds_dcfl = 0 while (while_counter + 0.01) > Goal_Loss and iter <= args.epochs: data_DCFL['round_{}'.format(iter)] = [] Loss_local_each_global = [] loss_workers = np.zeros((args.num_users, args.epochs)) w_locals, loss_locals = [], [] m = max(int(args.frac * args.num_users), 1) idxs_users = np.random.choice(range(args.num_users), m, replace=False) counter_threshold = 0 print(iter, " in dist FL started") #if iter % 5 == 0: x = copy.deepcopy(net_glob) x.eval() acc_test_global, loss_test_global = test_img(x, valid_ds, args) Loss_local_each_global_total.append(acc_test_global) Global_Accuracy_Tracker[iter] = acc_test_global Global_Loss_Tracker[iter] = loss_test_global if iter > 0 & (Global_Loss_Tracker[iter-1] - Global_Loss_Tracker[iter] <= beta): threshold = threshold - gamma if threshold == 0.0: threshold = 1.0 print("threshold decreased to", threshold) workers_count = 0 SD_acc_full_distributed.append(acc_test_global) SD_loss_full_ditributed.append(loss_test_global) temp_w_locals = [] temp_workers_loss = np.empty(args.num_users) temp_workers_accuracy = np.empty(args.num_users) temp_workers_loss_test = np.empty(args.num_users) temp_workers_loss_differenc = np.empty(args.num_users) temp_workers_accuracy_differenc = np.empty(args.num_users) flag = np.zeros(args.num_users) list_of_random_workers_newfl = [] if iter < (args.epochs/2): for key, value in dict_workers_index.items(): # print(value) if key == iter: list_of_random_workers_newfl = dict_workers_index[key] else: list_of_random_workers_newfl = random.sample(workers, m) for idx in list_of_random_workers_newfl: #print("train started") # before starting train temp_before = copy.deepcopy(net_glob) # temp_before.load_state_dict(w) temp_before.eval() acc_test_local_before, loss_test_local_before = test_img(temp_before, valid_ds, args) local = LocalUpdate(args=args, dataset=dataset_train, idxs=dict_users[idx]) w, loss = local.train(net=copy.deepcopy(net_glob).to(args.device)) #print(w) #print("train completed") #print("type of idx is ", type(temp_w_locals)) temp_w_locals.append(copy.deepcopy(w)) temp_workers_loss[idx] = copy.deepcopy(loss) temp_after = copy.deepcopy(net_glob) temp_after.load_state_dict(w) temp_after.eval() acc_test_local_after, loss_test_local_after = test_img(temp_after, valid_ds, args) loss_workers_total[idx, iter] = loss_test_local_after temp_workers_accuracy[idx] = acc_test_local_after temp_workers_loss_test[idx] = loss_test_local_after temp_workers_loss_differenc[idx] = loss_test_local_before - loss_test_local_after temp_workers_accuracy_differenc[idx] = acc_test_local_after - acc_test_local_before print("train finished") while len(w_locals) < 1: #print("recieving started") index = 0 for idx in list_of_random_workers_newfl: #print("acc is ", temp_workers_accuracy[idx]) # print(temp_workers_loss_differenc) if workers_count >= m: break elif temp_workers_loss_differenc[idx] >= (threshold) \ and temp_workers_loss_differenc[idx] > 0 \ and flag[idx]==0: print("Update Received") w_locals.append(copy.deepcopy(temp_w_locals[index])) #print(temp_w_locals[index]) loss_locals.append(temp_workers_loss[idx]) flag[idx] = 1 workers_count += 1 workers_participation[idx][iter] = 1 data_DCFL['round_{}'.format(iter)].append({ 'C': args.frac, 'User ID': idx, 'Loss Train': loss_workers_total[idx, iter], 'Loss Test': temp_workers_loss[idx], 'Accuracy': temp_workers_accuracy[idx] }) index += 1 if len(w_locals) < 1: threshold = threshold / 2 if threshold == -np.inf: threshold = 1 print("threshold increased to ", threshold) # update global weights w_glob = FedAvg(w_locals) # copy weight to net_glob net_glob.load_state_dict(w_glob) #print("round completed") loss_avg = sum(loss_locals) / len(loss_locals) loss_train.append(loss_avg) workers_percent_dist.append(workers_count/args.num_users) counter_threshold_decrease[iter] = counter_threshold print(iter, " round dist fl finished") acc_test_final, loss_test_final = test_img(net_glob, dataset_test, args) while_counter = loss_test_final data_Global_DCFL["Round"].append(iter) data_Global_DCFL["C"].append(args.frac) data_Global_DCFL["Average Loss Train"].append(loss_avg) data_Global_DCFL["Accuracy Test"].append(Global_Accuracy_Tracker[iter]) data_Global_DCFL["Average Loss Test"].append(Global_Loss_Tracker[iter]) data_Global_DCFL["Workers Number"].append(workers_count) data_Global_DCFL["Large Test Loss"].append(float(loss_test_final)) data_Global_DCFL["Large Test Accuracy"].append(float(acc_test_final)) total_rounds_dcfl = iter iter = iter + 1 #plot workers percent of participating workers_percent_final = np.zeros(args.num_users) workers_name = np.empty(args.num_users) #print(workers_participation) for i in range(len(workers_participation[:, 1])): workers_percent_final[i] = sum(workers_participation[i, :])/args.epochs workers_name[i] = i workers_fraction_each_round_newFL = sum(workers_percent_final)/len(workers_percent_final) # testing #print("testing started") net_glob.eval() #print("train test started") acc_train_final, loss_train_final = test_img(net_glob, dataset_train, args) #print("train test finished") acc_test_final, loss_test_final = test_img(net_glob, dataset_test, args) acc_full_distributed.append(acc_test_final) loss_full_ditributed.append(loss_test_final) Final_LargeDataSetTest_DCFL["C"].append(args.frac) Final_LargeDataSetTest_DCFL["Test Loss"].append(float(loss_test_final)) Final_LargeDataSetTest_DCFL["Test Accuracy"].append(float(acc_test_final)) Final_LargeDataSetTest_DCFL["Train Loss"].append(float(loss_train_final)) Final_LargeDataSetTest_DCFL["Train Accuracy"].append(float(acc_train_final)) Final_LargeDataSetTest_DCFL["Total Rounds"].append(int(total_rounds_dcfl)) variable_start = variable_start + while_counter print("C is ", c_counter/10) with open('CIFAR_100users_data_main_1229-2020.json', 'w') as outfile: json.dump(data_main, outfile) with open('CIFAR_100users_data_DCFL_1229-2020.json', 'w') as outfile: json.dump(data_DCFL, outfile) with open('CIFAR_100users_data_DCFL_Global_1229-2020.json', 'w') as outfile: json.dump(data_Global_DCFL, outfile) with open('CIFAR_100users_data_main_Global_1229-2020.json', 'w') as outfile: json.dump(data_Global_main, outfile) with open('Final-CIFAR_100users_data_main_Global_1229-2020.json', 'w') as outfile: json.dump(Final_LargeDataSetTest_MainFL, outfile) with open('Final-CIFAR_100users_data_DCFL_Global_1229-2020.json', 'w') as outfile: json.dump(Final_LargeDataSetTest_DCFL, outfile) return 1
writer.add_histogram(f'all_user/data_distribution', np.array(dataset_train.targets)[v], bins=np.arange(11), global_step=k) # build model if args.model == 'lenet' and (args.dataset == 'cifar' or args.dataset == 'fmnist'): net_glob = CNNCifar(args=args).to(args.device) elif args.model == 'vgg' and args.dataset == 'cifar': net_glob = vgg16().to(args.device) else: exit('Error: unrecognized model') print(net_glob) net_glob.train() # copy weights w_glob = net_glob.state_dict() # training loss_train = [] cv_loss, cv_acc = [], [] val_loss_pre, counter = 0, 0 net_best = None best_loss = None val_acc_list, net_list = [], [] test_best_acc = 0.0 for iter in range(args.epochs): w_locals, loss_locals = [], [] m = max(int(args.frac * args.num_users), 1) idxs_users = np.random.choice(range(args.num_users), m, replace=False) for idx in idxs_users:
net_local = CNNMnist(args=args).to(args.device) elif args.model == 'mlp': len_in = 1 for x in img_size: len_in *= x net_local = MLP(dim_in=len_in, dim_hidden=64, dim_out=args.num_classes).to(args.device) else: exit('Error: unrecognized model') #Let's train the model few epochs first n_epochs = 3 batch_size_train = 64 batch_size_test = 1000 learning_rate = 0.01 momentum = 0.5 log_interval = 10 random_seed = 1 torch.backends.cudnn.enabled = False torch.manual_seed(random_seed) data_weight = 0 w = net_local.state_dict() checkpoint = {'data_weight': data_weight, 'state_dict': w} torch.save(checkpoint, './BaseModel.pth')