def get_model_config(folder_configs,
dataset,
loss_func,
is_agg=False,
party_id=0):
if is_agg:
return None
model = SGDClassifier(loss='log', penalty='l2')
if dataset == 'adult':
model.classes_ = np.array([0, 1])
elif dataset == 'mnist':
model.classes_ = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
if not os.path.exists(folder_configs):
os.makedirs(folder_configs)
fname = os.path.join(folder_configs, 'model_architecture.pickle')
with open(fname, 'wb') as f:
pickle.dump(model, f)
# Generate model spec:
spec = {'model_definition': fname}
model = {
'name': 'SklearnSGDFLModel',
'path': 'ibmfl.model.sklearn_SGD_linear_fl_model',
'spec': spec
}
return model
def test(X, y, coef, intercept):
clf = SGDClassifier(fit_intercept=True)
clf.coef_ = coef.todense()
clf.intercept_ = intercept.todense()
clf.classes_ = np.unique(y)
return clf.score(X, y)
def _generate_pfa_classifier(result, indep_vars, featurizer, classes):
# Create mock SGDRegressor for sklearn_to_pfa
estimator = SGDClassifier()
estimator.classes_ = classes
estimator.intercept_ = [result[cat]['intercept'] for cat in classes]
# NOTE: linearly dependent columns will be assigned 0
estimator.coef_ = [[
result[cat].get(c, {'coef': 0.})['coef'] for c in featurizer.columns
if c != 'intercept'
] for cat in classes]
types = [(var['name'], var['type']['name']) for var in indep_vars]
return sklearn_to_pfa(estimator, types, featurizer.generate_pretty_pfa())
def server_update(
init_weight,
client_fraction,
num_rounds,
features,
labels,
epoch,
batch_size,
display_weight_per_round,
):
"""
Calls clientUpdate to get the updated weights from clients, and applies Federated
Averaging Algorithm to update the weight on server side
init_weights: weights to initialize the training with
ex: [weights of size num_classes*num_features, intercepts of size num_classes]
client_fraction: fraction of clients to use per round
num_rounds: number of rounds used to update the weight
features: a 3D array containing features for each sample
ex: [[[feature1, feature2], [feature1, feature2], ...]]
label: an array containing the labels for the corresponding sample in "features"
ex: [label1, label2, ...]
epoch: number of epochs to run the training for
batch_size: the size of each batch of data while training
display_weight_per_round: a boolean value used to toggle the display of weight value per round
"""
# initialize the weights
coef = list(init_weight[0])
intercept = list(init_weight[1])
# number of clients
client_num = len(features)
# fraction of clients
C = client_fraction
# use to generate n_k so that the sum of n_k equals to n
for i in range(num_rounds):
# calculate the number of clients used in this round
m = max(int(client_num * C), 1)
# random set of m client's index
S = np.array(random.sample(range(client_num), m))
num_samples = []
# grab all the weights from clients
client_coefs = None
client_intercepts = None
for i in S:
client_feature = features[i]
client_label = labels[i]
coefs, intercept = client_update([coef, intercept], epoch,
batch_size, client_feature,
client_label)
client_coefs = append(
client_coefs,
coefs,
)
client_intercepts = append(client_intercepts, intercept)
num_samples.append(len(client_feature))
# calculate the new server weights based on new weights coming from client
new_coefs = np.zeros(init_weight[0].shape, dtype=np.float64, order="C")
new_intercept = np.zeros(init_weight[1].shape,
dtype=np.float64,
order="C")
for i in range(len(client_coefs)):
client_coef = client_coefs[i]
client_intercept = client_intercepts[i]
n_k = len(features[i])
added_coef = [
value * (n_k) / sum(num_samples) for value in client_coef
]
added_intercept = [
value * (n_k) / sum(num_samples) for value in client_intercept
]
new_coefs = np.add(new_coefs, added_coef)
new_intercept = np.add(new_intercept, added_intercept)
# update the server weights to newly calculated weights
coef = new_coefs
intercept = new_intercept
if display_weight_per_round:
print("Updated Weights: ", coef, intercept)
# load coefficients and intercept into the classifier
clf = SGDClassifier(loss="hinge", penalty="l2")
clf.coef_ = new_coefs
clf.intercept_ = new_intercept
clf.classes_ = np.unique(
list(labels)) # the unique labels are the classes for the classifier
return clf
def _train(train_data,
valid_data,
costs,
importance,
max_iter=10,
alpha=0.1,
minibatch=1000,
epochs=10,
l1_ratio=1.0,
penalty='none',
eta0=0.01):
"""Train one cost-aware linear model using SGD.
Args:
max_iter: number of passes over the mini-batch (mini-epoch?)
alpha: regularizer weight
minibatch: size of a mini-batch
epochs: number of passes over the training data
"""
x_train, y_train, qid_train = train_data
x_valid, y_valid, qid_valid = valid_data
model = SGDClassifier(alpha=alpha,
verbose=False,
shuffle=False,
n_iter=max_iter,
learning_rate='constant',
penalty=penalty,
l1_ratio=l1_ratio,
eta0=eta0)
model.classes_ = np.array([-1, 1])
# fit SGD over the full data to initialize the model weights
model.fit(x_train, y_train)
valid_scores = (np.nan, np.nan, np.nan)
if x_valid is not None:
m = test_all(model.decision_function(x_valid), y_valid, qid_valid, 1)
valid_scores = (m['ndcg@10'], m['p@10'], m['err@10'])
print(
'[%3i]: weighted L1 %8.2f, cost %8d, features %4d, valid ndcg@10/p@10/err@10 %0.4f/%0.4f/%0.4f'
% (0, np.sum(np.abs(model.coef_[0] * costs)),
np.sum(costs[np.nonzero(
model.coef_[0])]), np.count_nonzero(model.coef_[0]),
valid_scores[0], valid_scores[1], valid_scores[2]))
# SGD algorithm (Tsuruoka et al., 2009)
u = np.zeros(x_train.shape[1])
q = np.zeros(x_train.shape[1])
for epoch in range(1, epochs + 1):
for iterno, batch in enumerate(
batch_generator(x_train, y_train, minibatch, x_train.shape[0]),
1):
x, y = batch
# call the internal method to specify custom classes, coef_init, and intercept_init
model._partial_fit(x,
y,
alpha=model.alpha,
C=1.0,
loss=model.loss,
learning_rate=model.learning_rate,
n_iter=1,
classes=model.classes_,
sample_weight=None,
coef_init=model.coef_,
intercept_init=model.intercept_)
new_w = np.zeros(model.coef_.shape[1])
u += model.eta0 * model.alpha * costs / float(
x_train.shape[0]) # note the costs
for i in range(len(model.coef_[0])):
if model.coef_[0][i] > 0:
new_w[i] = max(0, model.coef_[0][i] - (u[i] + q[i]))
elif model.coef_[0][i] < 0:
new_w[i] = min(0, model.coef_[0][i] + (u[i] - q[i]))
q += new_w - model.coef_[0]
model.coef_[0] = new_w
valid_scores = (np.nan, np.nan, np.nan)
if x_valid is not None:
m = test_all(model.decision_function(x_valid), y_valid, qid_valid,
1)
valid_scores = (m['ndcg@10'], m['p@10'], m['err@10'])
print(
'[%3i]: weighted L1 %8.2f, cost %8d, features %4d, valid ndcg@10/p@10/err@10 %0.4f/%0.4f/%0.4f'
% (epoch, np.sum(np.abs(model.coef_[0] * costs)),
np.sum(costs[np.nonzero(
model.coef_[0])]), np.count_nonzero(model.coef_[0]),
valid_scores[0], valid_scores[1], valid_scores[2]))
return model
def f(coef, intercept, classes):
s = SGDClassifier()
s.coef_ = coef
s.intercept_ = intercept
s.classes_ = classes
return s
def server_update(
init_weight,
client_fraction,
num_rounds,
features,
labels,
epoch,
batch_size,
display_weight_per_round,
rand_seed,
):
"""
Calls client_update to get the updated weights from clients, and applies Federated
Averaging Algorithm to update the weight on server side
init_weights: weights to initialize the training with
ex: [weights of size num_classes*num_features, intercepts of size num_classes]
client_fraction: fraction of clients to use per round
num_rounds: number of rounds used to update the weight
features: a 3D array containing features for each sample
ex: [[[feature1, feature2], [feature1, feature2], ...]]
labels: an array containing the labels for the corresponding sample in "features"
ex: [label1, label2, ...]
epoch: number of epochs to run the training for
batch_size: the size of each batch of data while training
display_weight_per_round: a boolean value used to toggle the display of weight value per round
rand_seed: a seed to use with any random number generation in order to get consistant results between runs
"""
# initialize the weights
coef = init_weight[0]
intercept = init_weight[1]
# unique classes in the dataset
all_classes = np.unique(labels)
# number of clients
client_num = len(features)
# fraction of clients
C = client_fraction
# reseed the rng each run
random.seed(rand_seed)
serv = server.ServerFacade(coef, intercept)
# use to generate n_k so that the sum of n_k equals to n
for i in range(num_rounds):
# calculate the number of clients used in this round
m = max(int(client_num * C), 1)
# random set of m client's index
user_ids = np.array(random.sample(range(client_num), m))
for user_id in user_ids:
client_features = features[user_id]
num_samples = len(client_features)
client_labels = labels[user_id]
coefs, intercept = client_update(
[coef, intercept],
epoch,
batch_size,
client_features,
client_labels,
all_classes,
rand_seed,
)
# this will get moved to the end of Client.update_and_submit_weights
payload = {
"coefs": coefs.tolist(),
"intercept": intercept.tolist(),
"num_samples": num_samples,
}
serv.ingest_client_data(json.dumps(payload))
coef, intercept = serv.compute_new_weights()
# TODO: extract down to end of function so that we can construct
# a new SGD using new coef+intercept data.
# Reconstruct a new classifier so that we can test the accuracy
# using new coef and intercept
# load coefficients and intercept into the classifier
clf = SGDClassifier(loss="log", random_state=rand_seed)
clf.coef_ = coef
clf.intercept_ = intercept
clf.classes_ = np.unique(
list(labels)) # the unique labels are the classes for the classifier
return clf
