for i in aux_class:
cc_local = LocalUpdate(args=args,
dataset=dataset_train,
label=label_train,
idxs=dict_ratio[i],
alpha=None,
size_average=True)
cc_w, cc_lo, cc_ac = cc_local.train(
net=copy.deepcopy(net_glob).to(args.device))
cc_net.append(copy.deepcopy(cc_w))
cc_loss.append(copy.deepcopy(cc_lo))
pos = outlier_detect(w_glob, cc_net, iter)
# labeling process and updating global model
w_glob_last = copy.deepcopy(w_glob)
w_glob = FedAvg(w_locals)
if args.loss == 'ratio':
ratio = torch.tensor(whole_determination(pos, w_glob_last, cc_net),
dtype=torch.float)
print(ratio)
# copy weight to net_glob
net_glob.load_state_dict(w_glob)
# print loss
loss_avg = sum(loss_locals) / len(loss_locals)
ac_avg = sum(ac_locals) / len(ac_locals)
print('Round {:3d}, Average loss {:.3f}, Accuracy {:.3f}\n'.format(
iter, loss_avg, ac_avg))
loss_train.append(loss_avg)
loss_locals_15.append(copy.deepcopy(loss15))
#20 MALICIOUS
w_locals_20.append(copy.deepcopy(w20))
loss_locals_20.append(copy.deepcopy(loss20))
#25 MALICIOUS
w_locals_25.append(copy.deepcopy(w25))
loss_locals_25.append(copy.deepcopy(loss25))
#30 MALICIOUS
w_locals_30.append(copy.deepcopy(w30))
loss_locals_30.append(copy.deepcopy(loss30))
# update global weights
w_glob = FedAvg(w_locals)
w_glob1 = FedAvg(w_locals_1)
w_glob5 = FedAvg(w_locals_5)
w_glob10 = FedAvg(w_locals_10)
w_glob15 = FedAvg(w_locals_15)
w_glob20 = FedAvg(w_locals_20)
w_glob25 = FedAvg(w_locals_25)
w_glob30 = FedAvg(w_locals_30)
# copy weight to net_glob
net_glob.load_state_dict(w_glob)
net_glob1.load_state_dict(w_glob1)
net_glob5.load_state_dict(w_glob5)
net_glob10.load_state_dict(w_glob10)
net_glob15.load_state_dict(w_glob15)
net_glob20.load_state_dict(w_glob20)
def run_all(clf_all1, clf_all2, adv_all1, adv_all2, adv_all3):
# parse args
args = args_parser()
args.device = torch.device('cuda:{}'.format(
args.gpu) if torch.cuda.is_available() and args.gpu != -1 else 'cpu')
# load ICU dataset and split users
# load ICU data set
X, y, Z = load_ICU_data('../fairness-in-ml/data/adult.data')
if not args.iid:
X = X[:30000]
y = y[:30000]
Z = Z[:30000]
n_points = X.shape[0]
n_features = X.shape[1]
n_sensitive = Z.shape[1]
# split into train/test set
(X_train, X_test, y_train, y_test, Z_train,
Z_test) = train_test_split(X,
y,
Z,
test_size=0.5,
stratify=y,
random_state=7)
# standardize the data
scaler = StandardScaler().fit(X_train)
scale_df = lambda df, scaler: pd.DataFrame(
scaler.transform(df), columns=df.columns, index=df.index)
X_train = X_train.pipe(scale_df, scaler)
X_test = X_test.pipe(scale_df, scaler)
class PandasDataSet(TensorDataset):
def __init__(self, *dataframes):
tensors = (self._df_to_tensor(df) for df in dataframes)
super(PandasDataSet, self).__init__(*tensors)
def _df_to_tensor(self, df):
if isinstance(df, pd.Series):
df = df.to_frame('dummy')
return torch.from_numpy(df.values).float()
def _df_to_tensor(df):
if isinstance(df, pd.Series):
df = df.to_frame('dummy')
return torch.from_numpy(df.values).float()
train_data = PandasDataSet(X_train, y_train, Z_train)
test_data = PandasDataSet(X_test, y_test, Z_test)
print('# train samples:', len(train_data)) # 15470
print('# test samples:', len(test_data))
batch_size = 32
train_loader = DataLoader(train_data,
batch_size=batch_size,
shuffle=True,
drop_last=True)
test_loader = DataLoader(test_data,
batch_size=len(test_data),
shuffle=True,
drop_last=True)
# sample users
if args.iid:
dict_users_train = fair_iid(train_data, args.num_users)
dict_users_test = fair_iid(test_data, args.num_users)
else:
train_data = [
_df_to_tensor(X_train),
_df_to_tensor(y_train),
_df_to_tensor(Z_train)
]
test_data = [
_df_to_tensor(X_test),
_df_to_tensor(y_test),
_df_to_tensor(Z_test)
]
#import pdb; pdb.set_trace()
dict_users_train, rand_set_all = fair_noniid(train_data,
args.num_users,
num_shards=100,
num_imgs=150,
train=True)
dict_users_test, _ = fair_noniid(test_data,
args.num_users,
num_shards=100,
num_imgs=150,
train=False,
rand_set_all=rand_set_all)
train_data = [
_df_to_tensor(X_train),
_df_to_tensor(y_train),
_df_to_tensor(Z_train)
]
test_data = [
_df_to_tensor(X_test),
_df_to_tensor(y_test),
_df_to_tensor(Z_test)
]
class LocalClassifier(nn.Module):
def __init__(self, n_features, n_hidden=32, p_dropout=0.2):
super(LocalClassifier, self).__init__()
self.network1 = nn.Sequential(nn.Linear(n_features, n_hidden),
nn.ReLU(), nn.Dropout(p_dropout),
nn.Linear(n_hidden, n_hidden),
nn.ReLU(), nn.Dropout(p_dropout),
nn.Linear(n_hidden, n_hidden))
self.network2 = nn.Sequential(nn.ReLU(), nn.Dropout(p_dropout),
nn.Linear(n_hidden, 1))
def forward(self, x):
mid = self.network1(x)
final = torch.sigmoid(self.network2(mid))
return mid, final
def pretrain_classifier(clf, data_loader, optimizer, criterion):
losses = 0.0
for x, y, _ in data_loader:
x = x.to(args.device)
y = y.to(args.device)
clf.zero_grad()
mid, p_y = clf(x)
loss = criterion(p_y, y)
loss.backward()
optimizer.step()
losses += loss.item()
print('loss', losses / len(data_loader))
return clf
def test_classifier(clf, data_loader):
losses = 0
assert len(data_loader) == 1
with torch.no_grad():
for x, y_test, _ in data_loader:
x = x.to(args.device)
mid, y_pred = clf(x)
y_pred = y_pred.cpu()
clf_accuracy = metrics.accuracy_score(y_test,
y_pred > 0.5) * 100
return clf_accuracy
class Adversary(nn.Module):
def __init__(self, n_sensitive, n_hidden=32):
super(Adversary, self).__init__()
self.network = nn.Sequential(
nn.Linear(n_hidden, n_hidden),
nn.ReLU(),
nn.Linear(n_hidden, n_hidden),
nn.ReLU(),
nn.Linear(n_hidden, n_hidden),
nn.ReLU(),
nn.Linear(n_hidden, n_sensitive),
)
def forward(self, x):
return torch.sigmoid(self.network(x))
def pretrain_adversary(adv, clf, data_loader, optimizer, criterion):
losses = 0.0
for x, _, z in data_loader:
x = x.to(args.device)
z = z.to(args.device)
mid, p_y = clf(x)
mid = mid.detach()
p_y = p_y.detach()
adv.zero_grad()
p_z = adv(mid)
loss = (criterion(p_z.to(args.device), z.to(args.device)) *
lambdas.to(args.device)).mean()
loss.backward()
optimizer.step()
losses += loss.item()
print('loss', losses / len(data_loader))
return adv
def test_adversary(adv, clf, data_loader):
losses = 0
adv_accuracies = []
assert len(data_loader) == 1
with torch.no_grad():
for x, _, z_test in data_loader:
x = x.to(args.device)
mid, p_y = clf(x)
mid = mid.detach()
p_y = p_y.detach()
p_z = adv(mid)
for i in range(p_z.shape[1]):
z_test_i = z_test[:, i]
z_pred_i = p_z[:, i]
z_pred_i = z_pred_i.cpu()
adv_accuracy = metrics.accuracy_score(
z_test_i, z_pred_i > 0.5) * 100
adv_accuracies.append(adv_accuracy)
return adv_accuracies
def train_both(clf, adv, data_loader, clf_criterion, adv_criterion,
clf_optimizer, adv_optimizer, lambdas):
# Train adversary
adv_losses = 0.0
for x, y, z in data_loader:
x = x.to(args.device)
z = z.to(args.device)
local, p_y = clf(x)
adv.zero_grad()
p_z = adv(local)
loss_adv = (adv_criterion(p_z.to(args.device), z.to(args.device)) *
lambdas.to(args.device)).mean()
loss_adv.backward()
adv_optimizer.step()
adv_losses += loss_adv.item()
print('adversarial loss', adv_losses / len(data_loader))
# Train classifier on single batch
clf_losses = 0.0
for x, y, z in data_loader:
pass
x = x.to(args.device)
y = y.to(args.device)
z = z.to(args.device)
local, p_y = clf(x)
p_z = adv(local)
clf.zero_grad()
if args.adv:
clf_loss = clf_criterion(p_y.to(args.device), y.to(
args.device)) - (
adv_criterion(p_z.to(args.device), z.to(args.device)) *
lambdas.to(args.device)).mean()
else:
clf_loss = clf_criterion(p_y.to(args.device), y.to(args.device))
clf_loss.backward()
clf_optimizer.step()
clf_losses += clf_loss.item()
print('classifier loss', clf_losses / len(data_loader))
return clf, adv
def eval_performance_text(test_loader_i, local_clf_i, adv_i):
with torch.no_grad():
for test_x, test_y, test_z in test_loader_i:
test_x = test_x.to(args.device)
local_pred, clf_pred = local_clf_i(test_x)
adv_pred = adv_i(local_pred)
y_post_clf = pd.Series(clf_pred.cpu().numpy().ravel(),
index=y_test[list(
dict_users_train[idx])].index)
Z_post_adv = pd.DataFrame(adv_pred.cpu().numpy(),
columns=Z_test.columns)
clf_roc_auc, clf_accuracy, adv_acc1, adv_acc2, adv_roc_auc = _performance_text(
test_y, test_z, y_post_clf, Z_post_adv, epoch=None)
return clf_roc_auc, clf_accuracy, adv_acc1, adv_acc2, adv_roc_auc
def eval_global_performance_text(test_loader_i, local_clf_i, adv_i,
global_clf):
with torch.no_grad():
for test_x, test_y, test_z in test_loader_i:
test_x = test_x.to(args.device)
local_pred, clf_pred = local_clf_i(test_x)
adv_pred = adv_i(local_pred)
global_pred = global_clf(local_pred)
y_post_clf = pd.Series(global_pred.cpu().numpy().ravel(),
index=y_test[list(
dict_users_train[idx])].index)
Z_post_adv = pd.DataFrame(adv_pred.cpu().numpy(),
columns=Z_test.columns)
clf_roc_auc, clf_accuracy, adv_acc1, adv_acc2, adv_roc_auc = _performance_text(
test_y, test_z, y_post_clf, Z_post_adv, epoch=None)
return clf_roc_auc, clf_accuracy, adv_acc1, adv_acc2, adv_roc_auc
lambdas = torch.Tensor([30.0, 30.0])
net_local_list = []
print(
'\n\n======================== STARTING LOCAL TRAINING ========================\n\n\n'
)
for idx in range(args.num_users):
print(
'\n======================== LOCAL TRAINING, USER %d ========================\n\n\n'
% idx)
train_data_i_raw = [
torch.FloatTensor(bb[list(dict_users_train[idx])])
for bb in train_data
]
train_data_i = TensorDataset(train_data_i_raw[0], train_data_i_raw[1],
train_data_i_raw[2])
train_loader_i = torch.utils.data.DataLoader(train_data_i,
batch_size=batch_size,
shuffle=False,
num_workers=4)
test_data_i_raw = [
torch.FloatTensor(bb[list(dict_users_train[idx])])
for bb in test_data
]
test_data_i = TensorDataset(test_data_i_raw[0], test_data_i_raw[1],
test_data_i_raw[2])
test_loader_i = torch.utils.data.DataLoader(
test_data_i,
batch_size=len(test_data_i),
shuffle=False,
num_workers=4)
local_clf_i = LocalClassifier(n_features=n_features).to(args.device)
local_clf_criterion_i = nn.BCELoss().to(args.device)
local_clf_optimizer_i = optim.SGD(local_clf_i.parameters(), lr=0.1)
adv_i = Adversary(Z_train.shape[1]).to(args.device)
adv_criterion_i = nn.BCELoss(reduce=False).to(args.device)
adv_optimizer_i = optim.SGD(adv_i.parameters(), lr=0.1)
net_local_list.append([
train_loader_i, test_loader_i, local_clf_i, local_clf_optimizer_i,
local_clf_criterion_i, adv_i, adv_criterion_i, adv_optimizer_i
])
N_CLF_EPOCHS = 10
for epoch in range(N_CLF_EPOCHS):
print(
'======================== pretrain_classifier epoch %d ========================'
% epoch)
local_clf = pretrain_classifier(local_clf_i, train_loader_i,
local_clf_optimizer_i,
local_clf_criterion_i)
# test classifier
# print ('\npretrained test accuracy on income prediction', test_classifier(local_clf_i, test_loader))
# print ()
print(
'======================== local classifier pretraining: evaluating _performance_text on device %d ========================'
% idx)
eval_performance_text(test_loader_i, local_clf_i, adv_i)
N_ADV_EPOCHS = 10
for epoch in range(N_ADV_EPOCHS):
print(
'======================== pretrain_adversary epoch %d ========================'
% epoch)
pretrain_adversary(adv_i, local_clf_i, train_loader_i,
adv_optimizer_i, adv_criterion_i)
# test adversary
# print ('\npretrained adversary accuracy on race, sex prediction', test_adversary(adv_i, local_clf_i, test_loader))
# print ()
print(
'======================== local adversary pretraining: evaluating _performance_text on device %d ========================'
% idx)
eval_performance_text(test_loader_i, local_clf_i, adv_i)
print(
'======================== by now both the local classifier and the local adversary should do well ========================'
)
# train both
N_EPOCH_COMBINED = 0 #250
for epoch in range(N_EPOCH_COMBINED):
print(
'======================== combined training epoch %d ========================'
% epoch)
clf, adv = train_both(local_clf_i, adv_i, train_loader_i,
local_clf_criterion_i, adv_criterion_i,
local_clf_optimizer_i, adv_optimizer_i,
lambdas)
# test classifier
#print ('final test accuracy on income prediction', test_classifier(clf, test_loader))
# test adversary
#print ('final adversary accuracy on race, sex prediction', test_adversary(adv, clf, test_loader))
print(
'======================== local classifier and adversary pretraining: evaluating _performance_text on device %d ========================'
% idx)
eval_performance_text(test_loader_i, local_clf_i, adv_i)
print(
'======================== by now the local classifier should do well but the local adversary should not do well ========================'
)
print(
'======================== done pretraining local classifiers and adversaries ========================'
)
class GlobalClassifier(nn.Module):
def __init__(self, n_hidden=32, p_dropout=0.2):
super(GlobalClassifier, self).__init__()
self.global_network = nn.Sequential(
nn.Linear(n_hidden, n_hidden),
nn.ReLU(),
nn.Dropout(p_dropout),
nn.Linear(n_hidden, n_hidden),
nn.ReLU(),
nn.Dropout(p_dropout),
nn.Linear(n_hidden, 1),
)
def forward(self, local):
final = torch.sigmoid(self.global_network(local))
return final
# build global model
global_clf = GlobalClassifier().to(args.device)
global_clf_criterion = nn.BCELoss().to(args.device)
global_clf_optimizer = optim.Adam(global_clf.parameters(), lr=0.01)
# copy weights
w_glob = global_clf.state_dict()
print(
'\n\n======================== STARTING GLOBAL TRAINING ========================\n\n\n'
)
global_epochs = 10
for iter in range(global_epochs):
w_locals, loss_locals = [], []
for idx in range(args.num_users):
print(
'\n\n======================== GLOBAL TRAINING, ITERATION %d, USER %d ========================\n\n\n'
% (iter, idx))
train_loader_i, test_loader_i, local_clf_i, local_clf_optimizer_i, local_clf_criterion_i, adv_i, adv_criterion_i, adv_optimizer_i = net_local_list[
idx]
# train both local models: classifier and adversary
if iter % 2 == 0:
N_EPOCH_COMBINED = 0 #65
for epoch in range(N_EPOCH_COMBINED):
print(
'======================== combined training epoch %d ========================'
% epoch)
local_clf_i, adv_i = train_both(local_clf_i, adv_i,
train_loader_i,
local_clf_criterion_i,
adv_criterion_i,
local_clf_optimizer_i,
adv_optimizer_i, lambdas)
local = LocalUpdate(args=args, dataset=train_loader_i)
w, loss = local.train(local_net=local_clf_i,
local_opt=local_clf_optimizer_i,
local_adv=adv_i,
adv_opt=adv_optimizer_i,
global_net=copy.deepcopy(global_clf).to(
args.device),
global_opt=global_clf_optimizer)
w_locals.append(copy.deepcopy(w))
loss_locals.append(copy.deepcopy(loss))
w_glob = FedAvg(w_locals)
# copy weight to net_glob
global_clf.load_state_dict(w_glob)
for idx in range(args.num_users):
train_loader_i, test_loader_i, local_clf_i, local_clf_optimizer_i, local_clf_criterion_i, adv_i, adv_criterion_i, adv_optimizer_i = net_local_list[
idx]
print(
'======================== local and global training: evaluating _performance_text on device %d ========================'
% idx)
eval_performance_text(test_loader_i, local_clf_i, adv_i)
print(
'======================== by now the local classifier should do well but the local adversary should not do well ========================'
)
print(
'======================== local and global training: evaluating _global_performance_text on device %d ========================'
% idx)
clf_roc_auc, clf_accuracy, adv_acc1, adv_acc2, adv_roc_auc = eval_global_performance_text(
test_loader_i, local_clf_i, adv_i, global_clf)
print(
'======================== by now the global classifier should work better than local classifier ========================'
)
clf_all1.append(clf_roc_auc)
clf_all2.append(clf_accuracy)
adv_all1.append(adv_acc1)
adv_all2.append(adv_acc2)
adv_all3.append(adv_roc_auc)
print('clf_all1', np.mean(np.array(clf_all1)), np.std(np.array(clf_all1)))
print('clf_all2', np.mean(np.array(clf_all2)), np.std(np.array(clf_all2)))
print('adv_all1', np.mean(np.array(adv_all1)), np.std(np.array(adv_all1)))
print('adv_all2', np.mean(np.array(adv_all2)), np.std(np.array(adv_all2)))
print('adv_all3', np.mean(np.array(adv_all3)), np.std(np.array(adv_all3)))
return clf_all1, clf_all2, adv_all1, adv_all2, adv_all3
x_value, count = np.unique(x_stat,
return_counts=True) # compute the P(Xm)
w_locals.append(copy.deepcopy(w)) # collect local model
loss_locals.append(
copy.deepcopy(loss)) #collect local loss fucntion
d_locals.extend(
d_local) # collect the isx of local training data in FL
beta_locals.append(np.max(beta)) # beta value
mu_locals.append(np.max(delta_bloss)) # mu value
sigma_locals.append(np.std(delta_bloss)) #sigma value
x_stat_loacals.append(x_stat) # Xm
pxm_locals.append(np.array(count / (np.sum(count)))) #P(Xm)
data_locals[iter] = d_locals #collect dta
w_glob_fl = FedAvg(w_locals) # update the global model
net_glob_fl.load_state_dict(w_glob_fl) # copy weight to net_glob
w_fl_iter.append(copy.deepcopy(w_glob_fl))
loss_fl = sum(loss_locals) / len(loss_locals)
loss_train_fl.append(loss_fl) # loss of FL
# compute P(Xg)
xg_value, xg_count = np.unique(x_stat_loacals, return_counts=True)
xg_count = np.array(xg_count) / (np.sum(xg_count))
print('fl,iter = ', iter, 'loss=', loss_fl)
# compute beta, mu, sigma
beta_max = (np.max(beta_locals))
mu_max = (np.max(mu_locals))
sigma_max = (np.max(sigma_locals))
non_malicious_structure10[iter][1][idx] = flattened_w10
#NO ATTACK
w_locals.append(copy.deepcopy(w))
loss_locals.append(copy.deepcopy(loss))
#5 MALICIOUS
w_locals_5.append(copy.deepcopy(w5))
loss_locals_5.append(copy.deepcopy(loss5))
#10 MALICIOUS
w_locals_10.append(copy.deepcopy(w10))
loss_locals_10.append(copy.deepcopy(loss10))
# update global weights
w_glob = FedAvg(w_locals)
w_glob_5 = FedAvg(w_locals_5)
w_glob_10 = FedAvg(w_locals_10)
# copy weight to net_glob
net_glob.load_state_dict(w_glob)
net_glob5.load_state_dict(w_glob_5)
net_glob10.load_state_dict(w_glob_10)
# print loss
loss_avg = sum(loss_locals) / len(loss_locals)
loss_avg_5 = sum(loss_locals_5) / len(loss_locals_5)
loss_avg_10 = sum(loss_locals_10) / len(loss_locals_10)
non_malicious_structure[iter][0] = loss_avg
non_malicious_structure5[iter][0] = loss_avg_5
loss_locals_15.append(copy.deepcopy(loss15))
#20 MALICIOUS
w_locals_20.append(copy.deepcopy(w20))
loss_locals_20.append(copy.deepcopy(loss20))
#25 MALICIOUS
w_locals_25.append(copy.deepcopy(w25))
loss_locals_25.append(copy.deepcopy(loss25))
#30 MALICIOUS
w_locals_30.append(copy.deepcopy(w30))
loss_locals_30.append(copy.deepcopy(loss30))
# update global weights
w_glob = FedAvg(w_locals)
w_glob_1 = FedAvg(w_locals_1)
w_glob_5 = FedAvg(w_locals_5)
w_glob_10 = FedAvg(w_locals_10)
w_glob_15 = FedAvg(w_locals_15)
w_glob_20 = FedAvg(w_locals_20)
w_glob_25 = FedAvg(w_locals_25)
w_glob_30 = FedAvg(w_locals_30)
# copy weight to net_glob
net_glob.load_state_dict(w_glob)
net_glob1.load_state_dict(w_glob_1)
net_glob5.load_state_dict(w_glob_5)
net_glob10.load_state_dict(w_glob_10)
net_glob15.load_state_dict(w_glob_15)
net_glob20.load_state_dict(w_glob_20)
