def test_federated_dataset_search(workers):
bob = workers["bob"]
alice = workers["alice"]
grid = sy.PrivateGridNetwork(*[bob, alice])
train_bob = th.Tensor(th.zeros(1000, 100)).tag("data").send(bob)
target_bob = th.Tensor(th.zeros(1000, 100)).tag("target").send(bob)
train_alice = th.Tensor(th.zeros(1000, 100)).tag("data").send(alice)
target_alice = th.Tensor(th.zeros(1000, 100)).tag("target").send(alice)
data = grid.search("data")
target = grid.search("target")
datasets = [
BaseDataset(data["bob"][0], target["bob"][0]),
BaseDataset(data["alice"][0], target["alice"][0]),
]
fed_dataset = sy.FederatedDataset(datasets)
train_loader = sy.FederatedDataLoader(fed_dataset,
batch_size=4,
shuffle=False,
drop_last=False)
counter = 0
for batch_idx, (data, target) in enumerate(train_loader):
counter += 1
assert counter == len(train_loader), f"{counter} == {len(fed_dataset)}"
def main():
hook = sy.TorchHook(torch)
device = torch.device("cpu")
model = Net()
model.build(torch.zeros([1, 1, 28, 28], dtype=torch.float).to(device))
# model.build(torch.zeros([1, node_num], dtype=torch.float).to(device))
@sy.func2plan()
def loss_fn(pred, target):
return nll_loss(input=pred, target=target)
input_num = torch.randn(3, 5, requires_grad=True)
target = torch.tensor([1, 0, 4])
dummy_pred = F.log_softmax(input_num, dim=1)
loss_fn.build(dummy_pred, target)
built_model = model
built_loss_fn = loss_fn
epoch_num = 21
batch_size = 64
lr = 0.1
learning_rate = lr
optimizer_args = {"lr": lr}
alice = NodeClient(hook, "ws://10.0.17.6:6666", id="alice")
# bob = NodeClient(hook, "ws://172.16.179.21:6667" , id="bob")
# charlie = NodeClient(hook, "ws://172.16.179.22:6668", id="charlie")
worker_list = [alice]
# worker_list = [alice]
grid = sy.PrivateGridNetwork(*worker_list)
for epoch in range(epoch_num):
logger.info("round %s/%s", epoch, epoch_num)
epoch_start = time.time()
jobs = [
gevent.spawn(send_model_to_worker, worker, built_model)
for worker in worker_list
]
gevent.joinall(jobs)
# results = await asyncio.gather(
# *[
# send_model_to_worker(
# worker=worker,
# built_model=built_model,
# )
# for worker in worker_list
# ]
# )
print("[PROF]", "AllWorkerSend", "duration", "COORD",
time.time() - epoch_start)
built_model.pointers = {}
built_loss_fn.pointers = {}
def distribute_dataset(self, X, y, train_idx, test_idx, workers):
tensor_X, tensor_y = torch.tensor(X).float(), torch.tensor(y).float()
split = int(
np.floor(self.model_config.validation_split * len(train_idx)))
train_idx, valid_idx = train_idx[split:], train_idx[:split]
indices = [train_idx, valid_idx, test_idx]
tags = ['train', 'valid', 'test']
for idx, tag in zip(indices, tags):
for i, (part_x, part_y) in enumerate(
self.node_distribution(tensor_X[idx], tensor_y[idx],
len(workers))):
tag_X = part_x.tag("#X", f"#{tag}").describe("")
tag_y = part_y.tag("#Y", f"#{tag}").describe("")
tag_X.send(workers[i], garbage_collect_data=False)
tag_y.send(workers[i], garbage_collect_data=False)
return sy.PrivateGridNetwork(*workers)
def test_virtual_grid(workers):
"""This tests our ability to simplify tuple types.
This test is pretty simple since tuples just serialize to
themselves, with a tuple wrapper with the correct ID (1)
for tuples so that the detailer knows how to interpret it."""
print(len(workers))
print(workers)
bob = workers["bob"]
alice = workers["alice"]
james = workers["james"]
grid = sy.PrivateGridNetwork(*[bob, alice, james])
x = torch.tensor([1, 2, 3, 4]).tag("#bob", "#male").send(bob)
y = torch.tensor([1, 2, 3, 4]).tag("#alice", "#female").send(alice)
z = torch.tensor([1, 2, 3, 4]).tag("#james", "#male").send(james)
results = grid.search()
assert len(results) == 3
assert "bob" in results.keys()
assert "alice" in results.keys()
assert "james" in results.keys()
results = grid.search("#bob")
assert len(results["bob"]) == 1
assert "alice" not in results
assert "james" not in results
results = grid.search("#male")
assert len(results["bob"]) == 1
assert "alice" not in results
assert len(results["james"]) == 1
async def main():
hook = sy.TorchHook(torch)
device = torch.device("cpu")
optimizer = "SGD"
epochs = 1
shuffle = True
model = Net()
model.build(torch.zeros([1, 1, 28, 28], dtype=torch.float).to(device))
# model.build(torch.zeros([2], dtype=torch.float).to(device))
@sy.func2plan(args_shape=[(-1, 1), (-1, 1)])
def loss_fn(target, pred):
return ((target.view(pred.shape).float() - pred.float())**2).mean()
batch_size = 64
lr = 0.1
learning_rate = lr
optimizer_args = {"lr": lr}
model_config = sy.ModelConfig(model=model,
loss_fn=loss_fn,
optimizer=optimizer,
batch_size=batch_size,
optimizer_args=optimizer_args,
epochs=epochs,
shuffle=shuffle)
# alice = NodeClient(hook, "ws://172.16.179.20:6666" , id="alice")
# bob = NodeClient(hook, "ws://172.16.179.21:6667" , id="bob")
# charlie = NodeClient(hook, "ws://172.16.179.22:6668", id="charlie")
# testing = NodeClient(hook, "ws://localhost:6669" , id="testing")
# worker_list = [alice, bob, charlie]
worker_list = []
for i in range(2, 2 + 12):
worker = NodeClient(hook,
"ws://" + flvm_ip[i] + ":6666",
id="flvm-" + str(i))
worker_list.append(worker)
for worker in worker_list:
model_config.send(worker)
grid = sy.PrivateGridNetwork(*worker_list)
num_of_parameters = len(model.parameters())
return_ids = []
for i in range(num_of_parameters):
return_ids.append("p" + str(i))
start = time.time()
# worker_0 = worker_list[0]
# worker_1 = worker_list[1]
# worker_2 = worker_list[2]
enc_results = await asyncio.gather(*[
worker.async_model_share(worker_list, return_ids=return_ids)
for worker in worker_list
])
end = time.time()
## aggregation
dst_enc_model = enc_results[0]
agg_start = time.time()
with torch.no_grad():
for i in range(len(dst_enc_model)):
layer_start = time.time()
for j in range(1, len(enc_results)):
add_start = time.time()
dst_enc_model[i] += enc_results[j][i]
print("[PROF]", "AddParams", time.time() - add_start)
print("[PROF]", "Layer" + str(i), time.time() - layer_start)
print("[PROF]", "AggTime", time.time() - agg_start)
async def main():
hook = sy.TorchHook(torch)
device = torch.device("cpu")
model = Net()
model.build(torch.zeros([1, 1, 28, 28], dtype=torch.float).to(device))
# model.build(torch.zeros([1, node_num], dtype=torch.float).to(device))
@sy.func2plan()
def loss_fn(pred, target):
return nll_loss(input=pred, target=target)
input_num = torch.randn(3, 5, requires_grad=True)
target = torch.tensor([1, 0, 4])
dummy_pred = F.log_softmax(input_num, dim=1)
loss_fn.build(dummy_pred, target)
built_model = model
built_loss_fn = loss_fn
epoch_num = 21
batch_size = 64
lr = 0.1
learning_rate = lr
optimizer_args = {"lr": lr}
if ssl_args == "ssl_true":
alice = NodeClient(hook, "wss://10.0.17.6:6666", id="alice")
else:
alice = NodeClient(hook, "ws://10.0.17.6:6666", id="alice")
# bob = NodeClient(hook, "ws://172.16.179.22:6667" , id="bob")
# charlie = NodeClient(hook, "ws://172.16.179.23:6668", id="charlie")
# med24 = NodeClient(hook, "ws://172.16.179.24:6669", id="med24")
# testing = NodeClient(hook, "ws://localhost:6669" , id="testing")
# worker_list = [alice, bob, charlie]
worker_list = [alice]
grid = sy.PrivateGridNetwork(*worker_list)
for epoch in range(epoch_num):
logger.info("round %s/%s", epoch, epoch_num)
for worker in worker_list:
built_model.id = "GlobalModel"
# built_loss_fn.id = "LossFunc"
# model_config = sy.ModelConfig(model=built_model,
# loss_fn=built_loss_fn,
# optimizer="SGD",
# batch_size=batch_size,
# optimizer_args={"lr": lr},
# epochs=1,
# max_nr_batches=-1)
model_send_start = time.time()
##pdb.set_trace()
built_model.send(worker)
model_send_end = time.time()
# print("[TEST]", "ModelSend", "time", model_send_start, model_send_end)
print("[trace] ModelSend duration", worker.id,
model_send_end - model_send_start)
built_model.pointers = {}
built_loss_fn.pointers = {}
# decay learning rate
learning_rate = max(0.98 * learning_rate, lr * 0.01)
def run(args):
if args.train:
print(f"Training over {args.epochs} epochs")
elif args.test:
print("Running a full evaluation")
else:
print("Running inference speed test")
print("model:\t\t", args.model)
print("dataset:\t", args.dataset)
print("batch_size:\t", args.batch_size)
hook = sy.TorchHook(torch)
if args.websockets:
alice = DataCentricFLClient(hook, "ws://localhost:7600")
bob = DataCentricFLClient(hook, "ws://localhost:7601")
crypto_provider = DataCentricFLClient(hook, "ws://localhost:7602")
my_grid = sy.PrivateGridNetwork(alice, bob, crypto_provider)
sy.local_worker.object_store.garbage_delay = 1
else:
bob = sy.VirtualWorker(hook, id="bob")
alice = sy.VirtualWorker(hook, id="alice")
crypto_provider = sy.VirtualWorker(hook, id="crypto_provider")
workers = [alice, bob]
sy.local_worker.clients = workers
encryption_kwargs = dict(workers=workers,
crypto_provider=crypto_provider,
protocol=args.protocol)
kwargs = dict(
requires_grad=args.requires_grad,
precision_fractional=args.precision_fractional,
dtype=args.dtype,
**encryption_kwargs,
)
if args.preprocess:
build_prepocessing(args.model, args.dataset, args.batch_size, workers,
args)
private_train_loader, private_test_loader = get_data_loaders(args,
kwargs,
private=True)
public_train_loader, public_test_loader = get_data_loaders(args,
kwargs,
private=False)
model = get_model(args.model,
args.dataset,
out_features=get_number_classes(args.dataset))
if args.test and not args.train:
load_state_dict(model, args.model, args.dataset)
model.eval()
if torch.cuda.is_available():
sy.cuda_force = True
if not args.public:
model.encrypt(**kwargs)
if args.fp_only: # Just keep the (Autograd+) Fixed Precision feature
model.get()
if args.train:
for epoch in range(args.epochs):
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum)
if not args.public:
optimizer = optimizer.fix_precision(
precision_fractional=args.precision_fractional,
dtype=args.dtype)
train_time = train(args, model, private_train_loader, optimizer,
epoch)
test_time, accuracy = test(args, model, private_test_loader)
else:
test_time, accuracy = test(args, model, private_test_loader)
if not args.test:
print(
f"{ 'Online' if args.preprocess else 'Total' } time (s):\t",
round(test_time / args.batch_size, 4),
)
else:
# Compare with clear text accuracy
print("Clear text accuracy is:")
model = get_model(args.model,
args.dataset,
out_features=get_number_classes(args.dataset))
load_state_dict(model, args.model, args.dataset)
test(args, model, public_test_loader)
if args.preprocess:
missing_items = [len(v) for k, v in sy.preprocessed_material.items()]
if sum(missing_items) > 0:
print("MISSING preprocessed material")
for key, value in sy.preprocessed_material.items():
print(f"'{key}':", value, ",")
async def main():
hook = sy.TorchHook(torch)
device = torch.device("cpu")
model = vgg.vgg16(pretrained=False)
# pdb.set_trace()
model.build(torch.zeros([64, 3, 32, 32], dtype=torch.float).to(device))
# pdb.set_trace()
@sy.func2plan()
def loss_fn(pred, target):
return nll_loss(input=pred, target=target)
input_num = torch.randn(3, 5, requires_grad=True)
target = torch.tensor([1, 0, 4])
dummy_pred = F.log_softmax(input_num, dim=1)
loss_fn.build(dummy_pred, target)
epoch_num = 11
batch_size = 64
lr = 0.05
learning_rate = lr
optimizer_args = {"lr": lr}
alice = NodeClient(hook, "ws://172.16.179.20:6666", id="alice")
bob = NodeClient(hook, "ws://172.16.179.21:6667", id="bob")
charlie = NodeClient(hook, "ws://172.16.179.22:6668", id="charlie")
# testing = NodeClient(hook, "ws://localhost:6669" , id="testing")
worker_list = [alice, bob, charlie]
grid = sy.PrivateGridNetwork(*worker_list)
for epoch in range(epoch_num):
logger.info("Training round %s/%s", epoch, epoch_num)
round_start_time = time.time()
results = await asyncio.gather(*[
fit_model_on_worker(
worker=worker,
built_model=model,
built_loss_fn=loss_fn,
encrypters=worker_list,
batch_size=batch_size,
curr_round=epoch,
max_nr_batches=-1,
lr=0.1,
) for worker in worker_list
])
local_train_end_time = time.time()
print("[trace]", "AllWorkersTrainingTime", "duration", "COORD",
local_train_end_time - round_start_time)
enc_models = {}
loss_values = {}
data_amounts = {}
total_data_amount = 0
for worker_id, enc_params, worker_loss, num_of_training_data in results:
if enc_params is not None:
enc_models[worker_id] = enc_params
loss_values[worker_id] = worker_loss
data_amounts[worker_id] = num_of_training_data
total_data_amount += num_of_training_data
## aggregation
nr_enc_models = len(enc_models)
enc_models_list = list(enc_models.values())
data_amounts_list = list(data_amounts.values()) ##
dst_enc_model = enc_models_list[0]
aggregation_start_time = time.time()
with torch.no_grad():
for i in range(len(dst_enc_model)):
for j in range(1, nr_enc_models):
dst_enc_model[i] += enc_models_list[j][i]
aggregation_end_time = time.time()
print("[trace]", "AggregationTime", "duration", "COORD",
aggregation_end_time - aggregation_start_time)
## decryption
new_params = []
decryption_start_time = time.time()
with torch.no_grad():
for i in range(len(dst_enc_model)):
decrypt_para = dst_enc_model[i].get()
new_para = decrypt_para.float_precision()
new_para = new_para / int(total_data_amount)
model.parameters()[i].set_(new_para)
round_end_time = time.time()
print("[trace]", "DecryptionTime", "duration", "COORD",
round_end_time - decryption_start_time)
print("[trace]", "RoundTime", "duration", "COORD",
round_end_time - round_start_time)
## FedAvg
# nr_models = len(models)
# model_list = list(models.values())
# dst_model = model_list[0]
# nr_params = len(dst_model.parameters())
# with torch.no_grad():
# for i in range(1, nr_models):
# src_model = model_list[i]
# src_params = src_model.parameters()
# dst_params = dst_model.parameters()
# for i in range(nr_params):
# dst_params[i].set_(src_params[i].data + dst_params[i].data)
# for i in range(nr_params):
# dst_params[i].set_(dst_params[i].data * 1/total_data_amount)
# if epoch%5 == 0 or epoch == 49:
# evaluate_model_on_worker(
# model_identifier="Federated model",
# worker=testing,
# dataset_key="mnist_testing",
# model=model,
# built_loss_fn=loss_fn,
# nr_bins=10,
# batch_size=64,
# device=device,
# print_target_hist=False,
# )
model.pointers = {}
loss_fn.pointers = {}
# decay learning rate
learning_rate = max(0.98 * learning_rate, lr * 0.01)
import pandas as pd
import streamlit as st
import webbrowser
hook = sy.TorchHook(torch)
# The local worker
me = hook.local_worker
me.is_client_worker = False
# The remote workers
bob = DataCentricFLClient(hook, "http://18.220.216.78:5001/")
#alice = DataCentricFLClient(hook, "ws://localhost:5006/")
# The crypto provider
sam = DataCentricFLClient(hook, "http://18.220.216.78:5001/")
kim = DataCentricFLClient(hook, "http://18.220.216.78:5001/")
grid = sy.PrivateGridNetwork(bob, sam, kim)
query = streamlit.text_input('Data Search Query')
data = grid.search(query)
nd = [[id, val[0].description, (list(val[0].shape)), val[0].tags]
for id, val in data.items()]
df = pd.DataFrame(nd, columns=['Location', 'Description', 'Size', 'Tags'])
streamlit.table(df)
values = df['Location'].tolist()
def genreate_url(id):
def main(data_path, participants, epochs, modelpath):
hook = sy.TorchHook(torch)
kwargs_websocket = {"host": "localhost", "hook": hook}
# kwargs_websocket = {"host": "localhost"}
device = torch.device("cuda" if use_cuda else "cpu")
workers = []
for participant in participants:
workers.append(sy.workers.websocket_client.WebsocketClientWorker(id=participant.id, port=participant.port, **kwargs_websocket))
# alice = sy.workers.websocket_client.WebsocketClientWorker(id="alice", port=8777, **kwargs_websocket)
# workers = [alice, bob, charlie]
grid = sy.PrivateGridNetwork(*workers)
data = grid.search("#mnist", "#data")
print(f"Search data: {len(data.keys())}")
target = grid.search("#mnist", "#target")
print(f"Search target: {len(target.keys())}")
# model = Net().to(device)
model_file = Path(modelpath)
if model_file.is_file():
model = torch.load(modelpath)
print(f"model loaded from file")
model.eval()
model = model.to(device)
else:
print(f"new model created")
model = Net().to(device)
print(f"model: {model}")
data = list(data.values())
target = list(target.values())
epoch_total = epoch_total_size(data)
print(f"Total epochs: {epoch_total}")
if use_cuda:
model.cuda()
optimizer = optim.Adadelta(model.parameters(), lr=learning_rate)
criterion = nn.CrossEntropyLoss()
test_loader = torch.utils.data.DataLoader(
datasets.MNIST(
data_path,
train=False,
transform=transforms.Compose(
[transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]
),
),
batch_size=1,
shuffle=True,
)
for epoch in range(1, epochs + 1):
m = train(epoch, model, data, target, optimizer, criterion)
test(m, device, test_loader)
torch.save(model, modelpath)
print(
'\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
test_loss, correct, len(test_loader.dataset),
100. * correct / len(test_loader.dataset)))
if __name__ == "__main__":
hook = sy.TorchHook(torch)
kwargs_websocket = {"host": "localhost", "hook": hook}
alice = WebsocketClientWorker(id="alice", port=8777, **kwargs_websocket)
device = torch.device("cuda" if use_cuda else "cpu")
# workers = [alice, bob, charlie]
workers = [alice]
grid = sy.PrivateGridNetwork(*workers)
data = grid.search("#mnist", "#data")
print(f"Search data: {len(data.keys())}")
target = grid.search("#mnist", "#target")
print(f"Search target: {len(target.keys())}")
datasets_my = []
for worker in data.keys():
dataset = sy.BaseDataset(data[worker][0], target[worker][0])
datasets_my.append(dataset)
n_features = data['alice'][0].shape[1]
n_targets = 1
