def __connect_with_node(self, node_id, node_url): if node_id not in self.hook.local_worker._known_workers: worker = DataCentricFLClient(self.hook, node_url) else: # There is already a connection to this node worker = self.hook.local_worker._known_workers[node_id] worker.connect() return worker
def connect_grid_nodes(message: dict) -> str: """Connect remote grid nodes between each other. Args: message (dict) : Dict data structure containing node_id, node address and user credentials(optional). Returns: response (str) : response message. """ if message["id"] not in local_worker._known_workers: worker = DataCentricFLClient(hook, address=message["address"], id=message["id"]) return json.dumps({"status": "Succesfully connected."})
def create_workers(self): if self.use_real_workers: addresses = [ f'ws://worker{worker_id}:{worker_id + 5000}/' for worker_id in range(self.num_of_workers) ] return [ DataCentricFLClient(self.hook, address) for address in addresses ] else: return [ sy.VirtualWorker(self.hook, id=worker_id) for worker_id in range(self.num_of_workers) ]
def run(args): if args.train: print(f"Training over {args.epochs} epochs") elif args.test: print("Running a full evaluation") else: print("Running inference speed test") print("model:\t\t", args.model) print("dataset:\t", args.dataset) print("batch_size:\t", args.batch_size) hook = sy.TorchHook(torch) if args.websockets: alice = DataCentricFLClient(hook, "ws://localhost:7600") bob = DataCentricFLClient(hook, "ws://localhost:7601") crypto_provider = DataCentricFLClient(hook, "ws://localhost:7602") my_grid = sy.PrivateGridNetwork(alice, bob, crypto_provider) sy.local_worker.object_store.garbage_delay = 1 else: bob = sy.VirtualWorker(hook, id="bob") alice = sy.VirtualWorker(hook, id="alice") crypto_provider = sy.VirtualWorker(hook, id="crypto_provider") workers = [alice, bob] sy.local_worker.clients = workers encryption_kwargs = dict(workers=workers, crypto_provider=crypto_provider, protocol=args.protocol) kwargs = dict( requires_grad=args.requires_grad, precision_fractional=args.precision_fractional, dtype=args.dtype, **encryption_kwargs, ) if args.preprocess: build_prepocessing(args.model, args.dataset, args.batch_size, workers, args) private_train_loader, private_test_loader = get_data_loaders(args, kwargs, private=True) public_train_loader, public_test_loader = get_data_loaders(args, kwargs, private=False) model = get_model(args.model, args.dataset, out_features=get_number_classes(args.dataset)) if args.test and not args.train: load_state_dict(model, args.model, args.dataset) model.eval() if torch.cuda.is_available(): sy.cuda_force = True if not args.public: model.encrypt(**kwargs) if args.fp_only: # Just keep the (Autograd+) Fixed Precision feature model.get() if args.train: for epoch in range(args.epochs): optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) if not args.public: optimizer = optimizer.fix_precision( precision_fractional=args.precision_fractional, dtype=args.dtype) train_time = train(args, model, private_train_loader, optimizer, epoch) test_time, accuracy = test(args, model, private_test_loader) else: test_time, accuracy = test(args, model, private_test_loader) if not args.test: print( f"{ 'Online' if args.preprocess else 'Total' } time (s):\t", round(test_time / args.batch_size, 4), ) else: # Compare with clear text accuracy print("Clear text accuracy is:") model = get_model(args.model, args.dataset, out_features=get_number_classes(args.dataset)) load_state_dict(model, args.model, args.dataset) test(args, model, public_test_loader) if args.preprocess: missing_items = [len(v) for k, v in sy.preprocessed_material.items()] if sum(missing_items) > 0: print("MISSING preprocessed material") for key, value in sy.preprocessed_material.items(): print(f"'{key}':", value, ",")
def create_websocket_client(hook, port, id): node = DataCentricFLClient(hook, "http://localhost:" + port, id=id) return node
import syft as sy from syft.grid.clients.data_centric_fl_client import DataCentricFLClient import torch as th hook = sy.TorchHook(th) bob = "http://bob:3000" hospital_datacluster = DataCentricFLClient(hook, bob) data_description = """Description: This data presents the monthly birth records. Columns: Gender: 0 - Male, 1 - Female Weight: in Kg Height: in cm Shape 5 * 3 """ monthly_birth_records = th.tensor([[1, 3.5, 47.3], [0, 3.7, 48.1], [0, 3.9, 50.0], [1, 4.1, 52.3], [0, 4.1, 49.7] ]) private_dataset = monthly_birth_records.private_tensor(allowed_users = ("Bob", "Ana", "Alice")) private_dataset = private_dataset.tag("#February", "#birth-records").describe(data_description) data_pointer = private_dataset.send(hospital_datacluster, user = "******")
import syft as sy from syft.grid.private_grid import PrivateGridNetwork from syft.grid.clients.data_centric_fl_client import DataCentricFLClient import torch as th hook = sy.TorchHook(th) bob = DataCentricFLClient(hook,'http://localhost:3000') grid = PrivateGridNetwork(bob) results = grid.search("#February", "#birth-records") feb_records = results['Bob'][0] def sum_column(dataset, column): sum_result = dataset[0][column].copy() for i in range(1, dataset.shape[0]): sum_result += dataset[i][column] return sum_result weight_sum = sum_column(feb_records, 1) avg_weight = weight_sum.get(user='******')/5 print(avg_weight)
import streamlit import torch import torch as th import syft as sy from syft.grid.clients.data_centric_fl_client import DataCentricFLClient import pandas as pd import streamlit as st import webbrowser hook = sy.TorchHook(torch) # The local worker me = hook.local_worker me.is_client_worker = False # The remote workers bob = DataCentricFLClient(hook, "http://18.220.216.78:5001/") #alice = DataCentricFLClient(hook, "ws://localhost:5006/") # The crypto provider sam = DataCentricFLClient(hook, "http://18.220.216.78:5001/") kim = DataCentricFLClient(hook, "http://18.220.216.78:5001/") grid = sy.PrivateGridNetwork(bob, sam, kim) query = streamlit.text_input('Data Search Query') data = grid.search(query) nd = [[id, val[0].description, (list(val[0].shape)), val[0].tags] for id, val in data.items()] df = pd.DataFrame(nd, columns=['Location', 'Description', 'Size', 'Tags']) streamlit.table(df)
def sweep( self, model, hook, model_id=None, mpc=False, allow_download=False, allow_remote_inference=False, apply=True, ): """ args: model : A jit model or Syft Plan. hook : A PySyft hook model_id (str): An integer/string representing the model id. If it isn't provided and the model is a Plan we use model.id, if the model is a jit model we raise an exception. allow_download (bool) : Allow to copy the model to run it locally. allow_remote_inference (bool) : Allow to run remote inferences. apply: to call terraform apply at the end """ print("Connecting to network-node") self.network_node = DataCentricFLClient(hook, self.gridnetwork_node_ip) print("Sending model to node") self.network_node.serve_model( model=model, model_id=model_id, mpc=mpc, allow_download=allow_download, allow_remote_inference=allow_remote_inference, ) print("Model sent, disconnecting node now") self.network_node.close() print("Node disconnected") with open("main.tf.json", "r") as main_config: self.config = json.load(main_config) if self.eviction_policy == "delete": if self.provider == "google": del self.config["resource"][ "google_compute_instance_group_manager"][self.cluster] del self.config["resource"][ "google_compute_instance_template"][self.template] if len(self.config["resource"] ["google_compute_instance_group_manager"]) == 0: del self.config["resource"][ "google_compute_instance_group_manager"] if len(self.config["resource"] ["google_compute_instance_template"]) == 0: del self.config["resource"][ "google_compute_instance_template"] with open("main.tf.json", "w") as main_config: json.dump(self.config, main_config, indent=2, sort_keys=False) if apply: if IPython.get_ipython(): terraform_notebook.apply() else: terraform_script.apply()