def main(config_path):
with open(config_path, 'r') as f:
config = yaml.load(f, Loader=yaml.FullLoader)
print(config)
device = torch.device(config['device'])
"""
Setup models
"""
global_model = MLP(dataset=config['dataset'],
num_layers=config['num_layers'],
hidden_size=config['hidden_size'],
drop_rate=config['drop_rate']).to(device)
print(global_model)
local_model_list = []
local_optim_list = []
for local_id in range(config['num_devices']):
local_model = MLP(dataset=config['dataset'],
num_layers=config['num_layers'],
hidden_size=config['hidden_size'],
drop_rate=config['drop_rate']).to(device)
local_optim = GD(local_model.parameters(), lr=config['lr'], weight_decay=1e-4)
local_model_list.append(local_model)
local_optim_list.append(local_optim)
personal_model_list = []
personal_optim_list = []
for local_id in range(config['num_devices']):
personal_model = MLP(dataset=config['dataset'],
num_layers=config['num_layers'],
hidden_size=config['hidden_size'],
drop_rate=config['drop_rate']).to(device)
personal_optim = GD(personal_model.parameters(), lr=config['lr'], weight_decay=1e-4)
personal_model_list.append(personal_model)
personal_optim_list.append(personal_optim)
init_weight = copy.deepcopy(global_model.state_dict())
criterion = nn.NLLLoss().to(device)
"""
load data and user group
"""
train_dataset, test_dataset, user_groups = get_dataset(config)
weight_per_user = [len(user_groups[local_id]) for local_id in range(config['num_devices'])]
weight_per_user = np.array(weight_per_user)/sum(weight_per_user)
trainloader_list, validloader_list = [], []
trainloader_iterator_list, validloader_iterator_list = [], []
for local_id in range(config['num_devices']):
trainloader, validloader = train_val_dataloader(train_dataset, user_groups[local_id],
batch_size=config['local_batch_size'])
trainloader_list.append(trainloader)
validloader_list.append(validloader)
trainloader_iterator_list.append(iter(trainloader_list[local_id]))
validloader_iterator_list.append(iter(validloader_list[local_id]))
"""
load initial value
"""
global_model.load_state_dict(init_weight)
for local_id in range(config['num_devices']):
local_model_list[local_id].load_state_dict(init_weight)
"""
start training
"""
global_acc = []
global_loss = []
local_loss = []
local_acc = []
train_loss = []
# test global model
list_acc, list_loss = [], []
for local_id in range(config['num_devices']):
acc, loss = inference(global_model, validloader_list[local_id], criterion, device)
list_acc.append(acc)
list_loss.append(loss)
global_acc += [sum(list_acc)/len(list_acc)]
global_loss += [sum(list_loss)/len(list_loss)]
print('global %d, acc %f, loss %f'%(len(global_acc), global_acc[-1], global_loss[-1]))
for global_iter in range(config['global_iters']): # T
activate_devices = np.random.permutation(config['num_devices'])[:config['num_active_devices']] # np.arange(config['num_devices'])
# get the local grad for each device
for local_iter in range(config['local_iters']): # E
cur_local_loss = []
cur_local_acc = []
cur_train_loss = []
for local_id in activate_devices: # K
# load single mini-batch
try:
inputs, labels = next(trainloader_iterator_list[local_id])
except StopIteration:
trainloader_iterator_list[local_id] = iter(trainloader_list[local_id])
inputs, labels = next(trainloader_iterator_list[local_id])
# train local model
inputs, labels = inputs.to(device), labels.to(device)
local_model_list[local_id].train()
local_model_list[local_id].zero_grad()
log_probs = local_model_list[local_id](inputs)
loss = criterion(log_probs, labels)
loss.backward()
# cur_train_loss.append(loss.item())
local_optim_list[local_id].step()
# train personalize model
personal_model_list[local_id].train()
personal_model_list[local_id].zero_grad()
log_probs_1 = local_model_list[local_id](inputs)
log_probs_2 = personal_model_list[local_id](inputs)
log_probs = (1-config['p_alpha']) * log_probs_1 + config['p_alpha'] * log_probs_2
loss = criterion(log_probs, labels)
loss.backward()
cur_train_loss.append(loss.item())
personal_optim_list[local_id].step()
# test local model
acc, loss = inference_personal(local_model_list[local_id], personal_model_list[local_id],
config['p_alpha'], validloader_list[local_id], criterion, device)
cur_local_loss.append(loss)
cur_local_acc.append(acc)
cur_local_loss = sum(cur_local_loss)/len(cur_local_loss)
cur_local_acc = sum(cur_local_acc)/len(cur_local_acc)
cur_train_loss = sum(cur_train_loss)/len(cur_train_loss)
local_loss.append(cur_local_loss)
local_acc.append(cur_local_acc)
train_loss.append(cur_train_loss)
# update learning rate
for local_id in activate_devices:
local_optim_list[local_id].inverse_prop_decay_learning_rate(global_iter)
# average local models
local_weight_list = [local_model.state_dict() for local_model in local_model_list]
avg_local_weight = average_state_dicts(local_weight_list, weight_per_user)
global_model.load_state_dict(avg_local_weight)
for local_id in range(config['num_devices']):
local_model_list[local_id].load_state_dict(avg_local_weight)
# test global model
list_acc, list_loss = [], []
for local_id in range(config['num_devices']):
acc, loss = inference(global_model, validloader_list[local_id], criterion, device)
list_acc.append(acc)
list_loss.append(loss)
global_acc += [sum(list_acc)/len(list_acc)]
global_loss += [sum(list_loss)/len(list_loss)]
print('global %d, acc %f, loss %f'%(len(global_acc), global_acc[-1], global_loss[-1]))
"""
save results
"""
with open('apfl_%s_noniid_%d.pkl'%(config['dataset'], config['noniid_level']), 'wb') as f:
pickle.dump([global_acc, global_loss, local_acc, local_loss, train_loss], f)
def main(config_path):
with open(config_path, 'r') as f:
config = yaml.load(f, Loader=yaml.FullLoader)
print(config)
device = torch.device(config['device'])
"""
Setup models
"""
global_model = Perceptron(dim_in=config['dimension'], dim_out=config['num_classes']).to(device)
local_model_list = []
local_optim_list = []
for local_id in range(config['num_devices']):
local_model = Perceptron(dim_in=config['dimension'], dim_out=config['num_classes']).to(device)
local_optim = GD(local_model.parameters(), lr=config['lr'], weight_decay=1e-4)
local_model_list.append(local_model)
local_optim_list.append(local_optim)
init_weight = copy.deepcopy(global_model.state_dict())
criterion = nn.NLLLoss().to(device)
"""
generate training data
"""
synthetic_dataset = SyntheticDataset(num_classes=config['num_classes'],
num_tasks=config['num_devices'],
num_dim=config['dimension'],
alpha=config['alpha'], beta=config['beta'])
data = synthetic_dataset.get_all_tasks()
num_samples = synthetic_dataset.get_num_samples()
weight_per_user = num_samples/sum(num_samples)
trainloader_list, validloader_list = [], []
trainloader_iterator_list, validloader_iterator_list = [], []
for local_id in range(config['num_devices']):
trainloader, validloader = train_val_dataloader(data[local_id]['x'], data[local_id]['y'], batch_size=config['local_batch_size'])
trainloader_list.append(trainloader)
validloader_list.append(validloader)
trainloader_iterator_list.append(iter(trainloader_list[local_id]))
validloader_iterator_list.append(iter(validloader_list[local_id]))
"""
load initial value
"""
global_model.load_state_dict(init_weight)
for local_id in range(config['num_devices']):
local_model_list[local_id].load_state_dict(init_weight)
"""
start training
"""
global_acc = []
global_loss = []
local_loss = []
local_acc = []
train_loss = []
# test global model
list_acc, list_loss = [], []
for local_id in range(config['num_devices']):
acc, loss = inference(global_model, validloader_list[local_id], criterion, device)
list_acc.append(acc)
list_loss.append(loss)
global_acc += [sum(list_acc)/len(list_acc)]
global_loss += [sum(list_loss)/len(list_loss)]
print('global %d, acc %f, loss %f'%(len(global_acc), global_acc[-1], global_loss[-1]))
for global_iter in range(config['global_iters']): # T
activate_devices = np.random.permutation(config['num_devices'])[:config['num_active_devices']] # np.arange(config['num_devices'])
# get the local grad for each device
for local_iter in range(config['local_iters']): # E
cur_local_loss = []
cur_local_acc = []
cur_train_loss = []
for local_id in activate_devices: # K
# load single mini-batch
try:
inputs, labels = next(trainloader_iterator_list[local_id])
except StopIteration:
trainloader_iterator_list[local_id] = iter(trainloader_list[local_id])
inputs, labels = next(trainloader_iterator_list[local_id])
# train local model
inputs, labels = inputs.to(device), labels.to(device)
local_model_list[local_id].train()
local_model_list[local_id].zero_grad()
log_probs = local_model_list[local_id](inputs)
loss = criterion(log_probs, labels)
loss.backward()
cur_train_loss.append(loss.item())
local_optim_list[local_id].step() # lr=config['lr']/(1+global_iter)
# test local model
acc, loss = inference(local_model_list[local_id], validloader_list[local_id], criterion, device)
cur_local_loss.append(loss)
cur_local_acc.append(acc)
cur_local_loss = sum(cur_local_loss)/len(cur_local_loss)
cur_local_acc = sum(cur_local_acc)/len(cur_local_acc)
cur_train_loss = sum(cur_train_loss)/len(cur_train_loss)
local_loss.append(cur_local_loss)
local_acc.append(cur_local_acc)
train_loss.append(cur_train_loss)
# update learning rate
for local_id in activate_devices:
local_optim_list[local_id].inverse_prop_decay_learning_rate(global_iter)
# average local models
local_weight_list = [local_model.state_dict() for local_model in local_model_list]
avg_local_weight = average_state_dicts(local_weight_list, weight_per_user)
global_model.load_state_dict(avg_local_weight)
for local_id in range(config['num_devices']):
local_model_list[local_id].load_state_dict(avg_local_weight)
# test global model
list_acc, list_loss = [], []
for local_id in range(config['num_devices']):
acc, loss = inference(global_model, validloader_list[local_id], criterion, device)
list_acc.append(acc)
list_loss.append(loss)
global_acc += [sum(list_acc)/len(list_acc)]
global_loss += [sum(list_loss)/len(list_loss)]
print('global %d, acc %f, loss %f'%(len(global_acc), global_acc[-1], global_loss[-1]))
"""
save results
"""
with open('fedavg_%.2f.pkl'%config['beta'], 'wb') as f:
pickle.dump([global_acc, global_loss, local_acc, local_loss, train_loss], f)
def startDescentGradient(self, idx_e1, idx_e2, r, er):
x = self.makeX(idx_e1, idx_e2)
y = self.makeY(idx_e1, idx_e2, r)
gd = GD(er, x, y)
result = gd.start()
return result
def main():
global N, Y, X_mu, X_S, flat_global_statistics_bounds, fix_beta, global_statistics_names
iterations = 100
# Load data
Y = numpy.concatenate((numpy.genfromtxt('./easydata/inputs/easy_1',
delimiter=','),
numpy.genfromtxt('./easydata/inputs/easy_2',
delimiter=','),
numpy.genfromtxt('./easydata/inputs/easy_3',
delimiter=','),
numpy.genfromtxt('./easydata/inputs/easy_4',
delimiter=',')))
N = Y.shape[0]
# We have several differet possible initialisations for the embeddings
#X_mu = numpy.load(X_file)
#X_mu = PCA(Y_file, Q)
#X_mu = scipy.randn(N, Q)
X_S = numpy.clip(
numpy.ones((N, Q)) * 0.5 + 0.01 * scipy.randn(N, Q), 0.001, 1)
#X_S = numpy.zeros((N, Q))
X_mu = numpy.concatenate(
(scipy.load('./easydata/embeddings/easy_1.embedding.npy'),
scipy.load('./easydata/embeddings/easy_2.embedding.npy'),
scipy.load('./easydata/embeddings/easy_3.embedding.npy'),
scipy.load('./easydata/embeddings/easy_4.embedding.npy')))
'''
X_S = numpy.concatenate((
scipy.load('./easydata/embeddings/easy_1.variance.npy'),
scipy.load('./easydata/embeddings/easy_2.variance.npy'),
scipy.load('./easydata/embeddings/easy_3.variance.npy'),
scipy.load('./easydata/embeddings/easy_4.variance.npy')))
'''
# Initialise the inducing points
Z = X_mu[numpy.random.permutation(N)[:M], :]
#Z = X_mu[:M,:]
Z += scipy.randn(M, Q) * 0.1
global_statistics_names = {
'Z': (M, Q),
'sf2': (1, 1),
'alpha': (1, Q),
'beta': (1, 1),
'X_mu': (N, Q),
'X_S': (N, Q)
}
# Initialise the global statistics
global_statistics = {
'Z': Z, # see GPy models/bayesian_gplvm.py
'sf2': numpy.array([[1.0]]), # see GPy kern/rbf.py
'alpha': scipy.ones((1, Q)), # see GPy kern/rbf.py
'beta': numpy.array([[1.0]]), # see GPy likelihood/gaussian.py
'X_Zmu': X_mu,
'X_S': X_S,
}
# Initialise bounds for optimisation
global_statistics_bounds = {
'Z': [(None, None) for i in range(M * Q)],
'sf2': [(0, None)],
'alpha': [(0, None) for i in range(Q)],
'beta': [(0, None)],
'X_mu': [(None, None) for i in range(N * Q)],
'X_S': [(0, None) for i in range(N * Q)],
}
flat_global_statistics_bounds = []
for key, statistic in global_statistics_bounds.items():
flat_global_statistics_bounds = flat_global_statistics_bounds + statistic
'''
Run the optimiser
'''
x0 = flatten_global_statistics(global_statistics)
# Transform the positiv parameters to be in the range (-Inf, Inf)
x0 = numpy.array([
transform_back(b, x) for b, x in zip(flat_global_statistics_bounds, x0)
])
'''
SCG optimisation (adapted from GPy implementation to reduce function calls)
The number of iterations might be greater than max_f_eval
'''
#fix_beta = True
# We set the flag fixed_embeddings to true because we just need the globals (which now include the embeddings) optimised
x = GD(likelihood_and_gradient,
x0,
'./easydata/tmp/',
fixed_embeddings=True,
display=True,
maxiters=iterations)
#fix_beta = False
#x = SC(likelihood_and_gradient, x[0], display=True, maxiters=iterations)
flat_array = x[0]
# Transform the parameters that have to be positive to be positive
flat_array_transformed = numpy.array([
transform(b, x)
for b, x in zip(flat_global_statistics_bounds, flat_array)
])
global_statistics = rebuild_global_statistics(global_statistics_names,
flat_array_transformed)
print 'Final global_statistics'
print global_statistics
class draw():
def __init__(self):
self.obj = None
self._range = 2 * math.pi
def show(self):
self.obj.fit()
plt.plot(self.obj.dg.X, self.obj.dg.Y,"or")
xs = np.linspace(0, self._range, 100)
f = lambda x:self._f(x);
plt.plot(xs, [math.sin(x) for x in xs])
plt.plot(xs, [f(x) for x in xs])
# plt.plot()
plt.show()
def _f(self,x):
coes = self.obj.coes.T.tolist()[0]
sum = 0
xx = 1
for coe in coes:
sum += xx * coe
xx *= x
return sum
if __name__ == '__main__':
draw = draw()
draw.obj = GD()
draw.show()