def __connect_with_node(self, node_id, node_url):
if node_id not in self.hook.local_worker._known_workers:
worker = DataCentricFLClient(self.hook, node_url)
else:
# There is already a connection to this node
worker = self.hook.local_worker._known_workers[node_id]
worker.connect()
return worker
def connect_grid_nodes(message: dict) -> str:
"""Connect remote grid nodes between each other.
Args:
message (dict) : Dict data structure containing node_id, node address and user credentials(optional).
Returns:
response (str) : response message.
"""
if message["id"] not in local_worker._known_workers:
worker = DataCentricFLClient(hook,
address=message["address"],
id=message["id"])
return json.dumps({"status": "Succesfully connected."})
def create_workers(self):
if self.use_real_workers:
addresses = [
f'ws://worker{worker_id}:{worker_id + 5000}/'
for worker_id in range(self.num_of_workers)
]
return [
DataCentricFLClient(self.hook, address)
for address in addresses
]
else:
return [
sy.VirtualWorker(self.hook, id=worker_id)
for worker_id in range(self.num_of_workers)
]
def run(args):
if args.train:
print(f"Training over {args.epochs} epochs")
elif args.test:
print("Running a full evaluation")
else:
print("Running inference speed test")
print("model:\t\t", args.model)
print("dataset:\t", args.dataset)
print("batch_size:\t", args.batch_size)
hook = sy.TorchHook(torch)
if args.websockets:
alice = DataCentricFLClient(hook, "ws://localhost:7600")
bob = DataCentricFLClient(hook, "ws://localhost:7601")
crypto_provider = DataCentricFLClient(hook, "ws://localhost:7602")
my_grid = sy.PrivateGridNetwork(alice, bob, crypto_provider)
sy.local_worker.object_store.garbage_delay = 1
else:
bob = sy.VirtualWorker(hook, id="bob")
alice = sy.VirtualWorker(hook, id="alice")
crypto_provider = sy.VirtualWorker(hook, id="crypto_provider")
workers = [alice, bob]
sy.local_worker.clients = workers
encryption_kwargs = dict(workers=workers,
crypto_provider=crypto_provider,
protocol=args.protocol)
kwargs = dict(
requires_grad=args.requires_grad,
precision_fractional=args.precision_fractional,
dtype=args.dtype,
**encryption_kwargs,
)
if args.preprocess:
build_prepocessing(args.model, args.dataset, args.batch_size, workers,
args)
private_train_loader, private_test_loader = get_data_loaders(args,
kwargs,
private=True)
public_train_loader, public_test_loader = get_data_loaders(args,
kwargs,
private=False)
model = get_model(args.model,
args.dataset,
out_features=get_number_classes(args.dataset))
if args.test and not args.train:
load_state_dict(model, args.model, args.dataset)
model.eval()
if torch.cuda.is_available():
sy.cuda_force = True
if not args.public:
model.encrypt(**kwargs)
if args.fp_only: # Just keep the (Autograd+) Fixed Precision feature
model.get()
if args.train:
for epoch in range(args.epochs):
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum)
if not args.public:
optimizer = optimizer.fix_precision(
precision_fractional=args.precision_fractional,
dtype=args.dtype)
train_time = train(args, model, private_train_loader, optimizer,
epoch)
test_time, accuracy = test(args, model, private_test_loader)
else:
test_time, accuracy = test(args, model, private_test_loader)
if not args.test:
print(
f"{ 'Online' if args.preprocess else 'Total' } time (s):\t",
round(test_time / args.batch_size, 4),
)
else:
# Compare with clear text accuracy
print("Clear text accuracy is:")
model = get_model(args.model,
args.dataset,
out_features=get_number_classes(args.dataset))
load_state_dict(model, args.model, args.dataset)
test(args, model, public_test_loader)
if args.preprocess:
missing_items = [len(v) for k, v in sy.preprocessed_material.items()]
if sum(missing_items) > 0:
print("MISSING preprocessed material")
for key, value in sy.preprocessed_material.items():
print(f"'{key}':", value, ",")
def create_websocket_client(hook, port, id):
node = DataCentricFLClient(hook, "http://localhost:" + port, id=id)
return node
import syft as sy
from syft.grid.clients.data_centric_fl_client import DataCentricFLClient
import torch as th
hook = sy.TorchHook(th)
bob = "http://bob:3000"
hospital_datacluster = DataCentricFLClient(hook, bob)
data_description = """Description:
This data presents the monthly birth records.
Columns:
Gender: 0 - Male, 1 - Female
Weight: in Kg
Height: in cm
Shape 5 * 3
"""
monthly_birth_records = th.tensor([[1, 3.5, 47.3],
[0, 3.7, 48.1],
[0, 3.9, 50.0],
[1, 4.1, 52.3],
[0, 4.1, 49.7]
])
private_dataset = monthly_birth_records.private_tensor(allowed_users = ("Bob", "Ana", "Alice"))
private_dataset = private_dataset.tag("#February", "#birth-records").describe(data_description)
data_pointer = private_dataset.send(hospital_datacluster, user = "******")
import syft as sy
from syft.grid.private_grid import PrivateGridNetwork
from syft.grid.clients.data_centric_fl_client import DataCentricFLClient
import torch as th
hook = sy.TorchHook(th)
bob = DataCentricFLClient(hook,'http://localhost:3000')
grid = PrivateGridNetwork(bob)
results = grid.search("#February", "#birth-records")
feb_records = results['Bob'][0]
def sum_column(dataset, column):
sum_result = dataset[0][column].copy()
for i in range(1, dataset.shape[0]):
sum_result += dataset[i][column]
return sum_result
weight_sum = sum_column(feb_records, 1)
avg_weight = weight_sum.get(user='******')/5
print(avg_weight)
import streamlit
import torch
import torch as th
import syft as sy
from syft.grid.clients.data_centric_fl_client import DataCentricFLClient
import pandas as pd
import streamlit as st
import webbrowser
hook = sy.TorchHook(torch)
# The local worker
me = hook.local_worker
me.is_client_worker = False
# The remote workers
bob = DataCentricFLClient(hook, "http://18.220.216.78:5001/")
#alice = DataCentricFLClient(hook, "ws://localhost:5006/")
# The crypto provider
sam = DataCentricFLClient(hook, "http://18.220.216.78:5001/")
kim = DataCentricFLClient(hook, "http://18.220.216.78:5001/")
grid = sy.PrivateGridNetwork(bob, sam, kim)
query = streamlit.text_input('Data Search Query')
data = grid.search(query)
nd = [[id, val[0].description, (list(val[0].shape)), val[0].tags]
for id, val in data.items()]
df = pd.DataFrame(nd, columns=['Location', 'Description', 'Size', 'Tags'])
streamlit.table(df)
def sweep(
self,
model,
hook,
model_id=None,
mpc=False,
allow_download=False,
allow_remote_inference=False,
apply=True,
):
"""
args:
model : A jit model or Syft Plan.
hook : A PySyft hook
model_id (str): An integer/string representing the model id.
If it isn't provided and the model is a Plan we use model.id,
if the model is a jit model we raise an exception.
allow_download (bool) : Allow to copy the model to run it locally.
allow_remote_inference (bool) : Allow to run remote inferences.
apply: to call terraform apply at the end
"""
print("Connecting to network-node")
self.network_node = DataCentricFLClient(hook, self.gridnetwork_node_ip)
print("Sending model to node")
self.network_node.serve_model(
model=model,
model_id=model_id,
mpc=mpc,
allow_download=allow_download,
allow_remote_inference=allow_remote_inference,
)
print("Model sent, disconnecting node now")
self.network_node.close()
print("Node disconnected")
with open("main.tf.json", "r") as main_config:
self.config = json.load(main_config)
if self.eviction_policy == "delete":
if self.provider == "google":
del self.config["resource"][
"google_compute_instance_group_manager"][self.cluster]
del self.config["resource"][
"google_compute_instance_template"][self.template]
if len(self.config["resource"]
["google_compute_instance_group_manager"]) == 0:
del self.config["resource"][
"google_compute_instance_group_manager"]
if len(self.config["resource"]
["google_compute_instance_template"]) == 0:
del self.config["resource"][
"google_compute_instance_template"]
with open("main.tf.json", "w") as main_config:
json.dump(self.config, main_config, indent=2, sort_keys=False)
if apply:
if IPython.get_ipython():
terraform_notebook.apply()
else:
terraform_script.apply()
class Cluster:
"""This class defines the Cluster object which is created in the method create_cluster"""
def __init__(self, name, provider, gridnetwork_ip, eviction_policy=None):
"""
args:
name: name of the cluster
provider: terrafrom provider for the cluster
gridnetwork_ip: ip of grid network instance
eviction_policy: "delete" to teardown the cluster after calling
cluster.sweep() else None
"""
self.name = name
self.provider = provider
self.gridnetwork_ip = gridnetwork_ip
self.gridnetwork_node_ip = gridnetwork_ip + ":3000"
self.master = name + "-master"
self.cluster = name + "-cluster"
self.template = name + "-template"
self.eviction_policy = eviction_policy
self.network_node = None
self.config = None
def sweep(
self,
model,
hook,
model_id=None,
mpc=False,
allow_download=False,
allow_remote_inference=False,
apply=True,
):
"""
args:
model : A jit model or Syft Plan.
hook : A PySyft hook
model_id (str): An integer/string representing the model id.
If it isn't provided and the model is a Plan we use model.id,
if the model is a jit model we raise an exception.
allow_download (bool) : Allow to copy the model to run it locally.
allow_remote_inference (bool) : Allow to run remote inferences.
apply: to call terraform apply at the end
"""
print("Connecting to network-node")
self.network_node = DataCentricFLClient(hook, self.gridnetwork_node_ip)
print("Sending model to node")
self.network_node.serve_model(
model=model,
model_id=model_id,
mpc=mpc,
allow_download=allow_download,
allow_remote_inference=allow_remote_inference,
)
print("Model sent, disconnecting node now")
self.network_node.close()
print("Node disconnected")
with open("main.tf.json", "r") as main_config:
self.config = json.load(main_config)
if self.eviction_policy == "delete":
if self.provider == "google":
del self.config["resource"][
"google_compute_instance_group_manager"][self.cluster]
del self.config["resource"][
"google_compute_instance_template"][self.template]
if len(self.config["resource"]
["google_compute_instance_group_manager"]) == 0:
del self.config["resource"][
"google_compute_instance_group_manager"]
if len(self.config["resource"]
["google_compute_instance_template"]) == 0:
del self.config["resource"][
"google_compute_instance_template"]
with open("main.tf.json", "w") as main_config:
json.dump(self.config, main_config, indent=2, sort_keys=False)
if apply:
if IPython.get_ipython():
terraform_notebook.apply()
else:
terraform_script.apply()