def sweep(self, apply=True):
"""
args:
apply: to call terraform apply at the end
"""
with open("main.tf.json", "r") as main_config:
self.config = json.load(main_config)
if self.eviction_policy == "delete":
if self.provider == "google":
del self.config["resource"][
"google_compute_instance_group_manager"][self.cluster]
del self.config["resource"][
"google_compute_instance_template"][self.template]
if len(self.config["resource"]
["google_compute_instance_group_manager"]) == 0:
del self.config["resource"][
"google_compute_instance_group_manager"]
if len(self.config["resource"]
["google_compute_instance_template"]) == 0:
del self.config["resource"][
"google_compute_instance_template"]
with open("main.tf.json", "w") as main_config:
json.dump(self.config, main_config, indent=2, sort_keys=False)
if apply:
if IPython.get_ipython():
terraform_notebook.apply()
else:
terraform_script.apply()
def compute_instance(self,
name,
machine_type,
zone,
image_family,
apply=True):
"""
args:
name: name of the compute instance
machine_type: the type of machine
zone: zone of your GCP project
image_family: image of the OS
apply: to call terraform apply at the end
"""
self.config += terrascript.resource.google_compute_instance(
name,
name=name,
machine_type=machine_type,
zone=zone,
boot_disk={"initialize_params": {
"image": image_family
}},
network_interface={
"network": "default",
"access_config": {}
},
)
with open("main.tf.json", "w") as main_config:
json.dump(self.config, main_config, indent=2, sort_keys=False)
if apply:
if IPython.get_ipython():
terraform_notebook.apply()
else:
terraform_script.apply()
def expose_port(self, name, ports=None, apply=True):
"""
args:
name: name of the resource
ports: list of ports to be exposed, defaults to 80
apply: to call terraform apply at the end
"""
if ports is not None:
ports = [80]
self.config += terrascript.resource.google_compute_firewall(
name,
name=name,
network="default",
allow={
"protocol": "tcp",
"ports": ports
},
)
with open("main.tf.json", "w") as main_config:
json.dump(self.config, main_config, indent=2, sort_keys=False)
if apply:
if IPython.get_ipython():
terraform_notebook.apply()
else:
terraform_script.apply()
def create_cluster(self,
name,
machine_type,
zone,
image_family,
target_size,
eviction_policy=None,
apply=True):
"""
args:
name: name of the compute instance
machine_type: the type of machine
zone: zone of your GCP project
image_family: image of the OS
target_size: number of wokers to be created(N workers + 1 master)
eviction_policy: "delete" to teardown the cluster after calling .sweep() else None
apply: to call terraform apply at the end
"""
if target_size < 3:
raise ValueError(
"The target-size should be equal to or greater than three.")
self.compute_instance(name + "-master", machine_type, zone,
image_family, False)
instance_template = terrascript.resource.google_compute_instance_template(
name + "-template",
name=name + "-template",
machine_type=machine_type,
disk={"source_image": image_family},
network_interface={
"network": "default",
"access_config": {}
},
lifecycle={"create_before_destroy": True},
)
self.config += instance_template
self.config += terrascript.resource.google_compute_instance_group_manager(
name + "-cluster",
name=name + "-cluster",
version={
"instance_template": "${" + instance_template.self_link + "}"
},
base_instance_name=name,
zone=zone,
target_size=str(target_size),
)
with open("main.tf.json", "w") as main_config:
json.dump(self.config, main_config, indent=2, sort_keys=False)
if apply:
if IPython.get_ipython():
terraform_notebook.apply()
else:
terraform_script.apply()
return Cluster(name, self.provider, eviction_policy=eviction_policy)
def create_gridnetwork(self, name, machine_type, zone, apply=True):
"""
args:
name: name of the compute instance
machine_type: the type of machine
zone: zone of your GCP project
apply: to call terraform apply at the end
"""
pygrid_network_ip = terrascript.resource.google_compute_address(name, name=name,)
self.config += pygrid_network_ip
self.config += terrascript.output(
name + "-ip", value="${" + pygrid_network_ip.address + "}",
)
self.expose_port(name="pygrid", apply=False)
image = terrascript.data.google_compute_image(
name + "pytorch",
family="pytorch-latest-gpu-debian-10",
project="deeplearning-platform-release",
)
self.config += image
node = terrascript.resource.google_compute_instance(
name,
name=name,
machine_type=machine_type,
zone=zone,
boot_disk={"initialize_params": {"image": "${" + image.self_link + "}"}},
network_interface={
"network": "default",
"access_config": {"nat_ip": "${" + pygrid_network_ip.address + "}"},
},
metadata_startup_script="""
#!/bin/bash
apt-get update
apt-get -y upgrade
sudo -i bash -c 'pip install git+https://github.com/OpenMined/PyGridNetwork.git'
sudo -i bash -c 'echo Starting PyGridNetwork & \
python -m gridnetwork --port=80 --start_local_db'""",
)
self.config += node
with open("main.tf.json", "w") as main_config:
json.dump(self.config, main_config, indent=2, sort_keys=False)
if apply:
if IPython.get_ipython():
terraform_notebook.apply()
else:
terraform_script.apply()
def create_gridnode(self, name, machine_type, zone, gridnetwork_name=None, apply=True):
"""
args:
name: name of the compute instance
machine_type: the type of machine
zone: zone of your GCP project
gridnetwork_name: name of gridnetwork instance created
apply: to call terraform apply at the end
"""
if not gridnetwork_name:
self.expose_port(name="pygrid", ports=[80], apply=False)
with open("terraform.tfstate", "r") as out:
outputs = json.load(out)["outputs"]
gridnetwork_ip = outputs[gridnetwork_name + "-ip"]["value"]
pygrid_network_address = "--gateway_url=http://" + gridnetwork_ip if gridnetwork_ip else ""
image = terrascript.data.google_compute_image(
name + "pytorch",
family="pytorch-latest-gpu-debian-10",
project="deeplearning-platform-release",
)
self.config += image
node = terrascript.resource.google_compute_instance(
name,
name=name,
machine_type=machine_type,
zone=zone,
boot_disk={"initialize_params": {"image": "${" + image.self_link + "}"}},
network_interface={"network": "default", "access_config": {}},
metadata_startup_script=f"""
#!/bin/bash
apt-get update
apt-get -y upgrade
sudo -i bash -c 'pip install notebook==5.7.8'
sudo -i bash -c 'pip install git+https://github.com/OpenMined/PyGridNode.git'
sudo -i bash -c 'echo Starting Node {name} \
joined with PyGridNetwork at {gridnetwork_ip} & \
python -m gridnode --id={name} --port=80 {pygrid_network_address}'""",
)
self.config += node
with open("main.tf.json", "w") as main_config:
json.dump(self.config, main_config, indent=2, sort_keys=False)
if apply:
if IPython.get_ipython():
terraform_notebook.apply()
else:
terraform_script.apply()
def reserve_ip(self, name, apply=True):
"""
args:
name: name of the reversed ip
apply: to call terraform apply at the end
"""
pygrid_network_ip = terrascript.resource.google_compute_address(name, name=name,)
self.config += pygrid_network_ip
self.config += terrascript.output(
name + "-ip", value="${" + pygrid_network_ip.address + "}",
)
with open("main.tf.json", "w") as main_config:
json.dump(self.config, main_config, indent=2, sort_keys=False)
if apply:
if IPython.get_ipython():
terraform_notebook.apply()
else:
terraform_script.apply()
def create_cluster(
self,
name,
machine_type,
zone,
reserve_ip_name,
target_size,
eviction_policy=None,
apply=True,
):
"""
args:
name: name of the compute instance
machine_type: the type of machine
zone: zone of your GCP project
reserve_ip_name: name of the reserved ip created using reserve_ip
target_size: number of wokers to be created(N workers + 1 master)
eviction_policy: "delete" to teardown the cluster after calling .sweep() else None
apply: to call terraform apply at the end
"""
self.expose_port("pygrid", apply=False)
with open("terraform.tfstate", "r") as out:
outputs = json.load(out)["outputs"]
gridnetwork_ip = outputs[reserve_ip_name + "-ip"]["value"]
image = terrascript.data.google_compute_image(
name + "pytorch",
family="pytorch-latest-gpu-debian-10",
project="deeplearning-platform-release",
)
self.config += image
self.config += terrascript.resource.google_compute_instance(
name + "pygridnetwork",
name=name + "pygridnetwork",
machine_type=machine_type,
zone=zone,
boot_disk={"initialize_params": {"image": "${" + image.self_link + "}"}},
network_interface={"network": "default", "access_config": {"nat_ip": gridnetwork_ip}},
metadata_startup_script="""
#!/bin/bash
apt-get update
apt-get -y upgrade
sudo -i bash -c 'pip install git+https://github.com/OpenMined/PyGridNetwork.git'
sudo -i bash -c 'echo Starting PyGridNetwork & \
python -m gridnetwork --port=80 --start_local_db'""",
)
pygrid_network_address = "http://" + gridnetwork_ip
instance_template = terrascript.resource.google_compute_instance_template(
name + "-template",
name=name + "-template",
machine_type=machine_type,
disk={"source_image": "${" + image.self_link + "}"},
network_interface={"network": "default", "access_config": {}},
metadata_startup_script=f"""
#!/bin/bash
apt-get update
apt-get -y upgrade
sudo -i bash -c 'pip install notebook==5.7.8'
sudo -i bash -c 'pip install git+https://github.com/OpenMined/PyGridNode.git'
sudo -i bash -c 'echo Starting Node {name} \
joined with PyGridNetwork at {pygrid_network_address} & \
python -m gridnode --id={name} --port=80 \
--gateway_url={pygrid_network_address}'""",
lifecycle={"create_before_destroy": True},
)
self.config += instance_template
self.config += terrascript.resource.google_compute_instance_group_manager(
name + "-cluster",
name=name + "-cluster",
version={"instance_template": "${" + instance_template.self_link + "}"},
base_instance_name=name,
zone=zone,
target_size=str(target_size),
)
with open("main.tf.json", "w") as main_config:
json.dump(self.config, main_config, indent=2, sort_keys=False)
if apply:
if IPython.get_ipython():
terraform_notebook.apply()
else:
terraform_script.apply()
return Cluster(name, self.provider, gridnetwork_ip, eviction_policy=eviction_policy)
def create_gridnetwork(self, name, machine_type, zone, apply=True):
"""
args:
name: name of the compute instance
machine_type: the type of machine
zone: zone of your GCP project
apply: to call terraform apply at the end
"""
pygrid_network_ip = terrascript.resource.google_compute_address(
name,
name=name,
)
self.config += pygrid_network_ip
self.config += terrascript.output(
name + "-ip",
value="${" + pygrid_network_ip.address + "}",
)
self.expose_port(name="pygrid", apply=False)
image = terrascript.data.google_compute_image(
name + "container-optimized-os",
family="cos-81-lts",
project="cos-cloud",
)
self.config += image
node = terrascript.resource.google_compute_instance(
name,
name=name,
machine_type=machine_type,
zone=zone,
boot_disk={
"initialize_params": {
"image": "${" + image.self_link + "}"
}
},
network_interface={
"network": "default",
"access_config": {
"nat_ip": "${" + pygrid_network_ip.address + "}"
},
},
metadata_startup_script="""
#!/bin/bash
sleep 30;
docker pull openmined/grid-network:production;
docker run \
-e PORT=80 \
-e DATABASE_URL=sqlite:///databasenode.db \
--name gridnetwork\
-p 80:80 \
-d openmined/grid-network:production;""",
)
self.config += node
with open("main.tf.json", "w") as main_config:
json.dump(self.config, main_config, indent=2, sort_keys=False)
if apply:
if IPython.get_ipython():
terraform_notebook.apply()
else:
terraform_script.apply()
def sweep(
self,
model,
hook,
model_id=None,
mpc=False,
allow_download=False,
allow_remote_inference=False,
apply=True,
):
"""
args:
model : A jit model or Syft Plan.
hook : A PySyft hook
model_id (str): An integer/string representing the model id.
If it isn't provided and the model is a Plan we use model.id,
if the model is a jit model we raise an exception.
allow_download (bool) : Allow to copy the model to run it locally.
allow_remote_inference (bool) : Allow to run remote inferences.
apply: to call terraform apply at the end
"""
print("Connecting to network-node")
self.network_node = DataCentricFLClient(hook, self.gridnetwork_node_ip)
print("Sending model to node")
self.network_node.serve_model(
model=model,
model_id=model_id,
mpc=mpc,
allow_download=allow_download,
allow_remote_inference=allow_remote_inference,
)
print("Model sent, disconnecting node now")
self.network_node.close()
print("Node disconnected")
with open("main.tf.json", "r") as main_config:
self.config = json.load(main_config)
if self.eviction_policy == "delete":
if self.provider == "google":
del self.config["resource"][
"google_compute_instance_group_manager"][self.cluster]
del self.config["resource"][
"google_compute_instance_template"][self.template]
if len(self.config["resource"]
["google_compute_instance_group_manager"]) == 0:
del self.config["resource"][
"google_compute_instance_group_manager"]
if len(self.config["resource"]
["google_compute_instance_template"]) == 0:
del self.config["resource"][
"google_compute_instance_template"]
with open("main.tf.json", "w") as main_config:
json.dump(self.config, main_config, indent=2, sort_keys=False)
if apply:
if IPython.get_ipython():
terraform_notebook.apply()
else:
terraform_script.apply()
def create_cluster(
self,
name,
machine_type,
zone,
reserve_ip_name,
target_size,
eviction_policy=None,
apply=True,
):
"""
args:
name: name of the compute instance
machine_type: the type of machine
zone: zone of your GCP project
reserve_ip_name: name of the reserved ip created using reserve_ip
target_size: number of wokers to be created(N workers + 1 master)
eviction_policy: "delete" to teardown the cluster after calling
cluster.sweep() else None
apply: to call terraform apply at the end
"""
self.expose_port("pygrid", ports=[80, 3000], apply=False)
with open("terraform.tfstate", "r") as out:
outputs = json.load(out)["outputs"]
gridnetwork_ip = outputs[reserve_ip_name + "-ip"]["value"]
pygrid_network_address = "http://" + gridnetwork_ip
image = terrascript.data.google_compute_image(
name + "container-optimized-os",
family="cos-81-lts",
project="cos-cloud",
)
self.config += image
self.config += terrascript.resource.google_compute_instance(
name + "pygridnetwork",
name=name + "pygridnetwork",
machine_type=machine_type,
zone=zone,
boot_disk={
"initialize_params": {
"image": "${" + image.self_link + "}"
}
},
network_interface={
"network": "default",
"access_config": {
"nat_ip": gridnetwork_ip
}
},
metadata_startup_script=f"""
#!/bin/bash
sleep 30;
docker pull openmined/grid-network:production && \
docker run \
-e PORT=80 \
-e DATABASE_URL=sqlite:///databasenode.db \
--name gridnetwork \
-p 80:80 \
-d openmined/grid-network:production;
docker pull openmined/grid-node:production;
docker run \
-e NODE_ID={name+"-network-node"} \
-e HOST="$(hostname -I)" \
-e PORT=3000 \
-e NETWORK={pygrid_network_address} \
-e DATABASE_URL=sqlite:///databasenode.db \
--name gridnode \
-p 3000:3000 \
-d openmined/grid-node:production;""",
)
# HOST environment variable is set to external IP address
instance_template = terrascript.resource.google_compute_instance_template(
name + "-template",
name=name + "-template",
machine_type=machine_type,
disk={"source_image": "${" + image.self_link + "}"},
network_interface={
"network": "default",
"access_config": {}
},
metadata_startup_script=f"""
#!/bin/bash
sleep 30;
docker pull openmined/grid-node:production;
docker run \
-e NODE_ID={name} \
-e HOST="$(curl ifconfig.co)" \
-e PORT=80 \
-e NETWORK={pygrid_network_address} \
-e DATABASE_URL=sqlite:///databasenode.db \
--name gridnode \
-p 80:80 \
-d openmined/grid-node:production;""",
lifecycle={"create_before_destroy": True},
)
self.config += instance_template
self.config += terrascript.resource.google_compute_instance_group_manager(
name + "-cluster",
name=name + "-cluster",
version={
"instance_template": "${" + instance_template.self_link + "}"
},
base_instance_name=name,
zone=zone,
target_size=str(target_size),
)
with open("main.tf.json", "w") as main_config:
json.dump(self.config, main_config, indent=2, sort_keys=False)
if apply:
if IPython.get_ipython():
terraform_notebook.apply()
else:
terraform_script.apply()
return Cluster(
name,
self.provider,
gridnetwork_ip,
eviction_policy=eviction_policy,
)
def create_gridnode(self,
name,
machine_type,
zone,
gridnetwork_name=None,
apply=True):
"""
args:
name: name of the compute instance
machine_type: the type of machine
zone: zone of your GCP project
gridnetwork_name: name of gridnetwork instance created
apply: to call terraform apply at the end
"""
if not gridnetwork_name:
self.expose_port(name="pygrid", ports=[80], apply=False)
with open("terraform.tfstate", "r") as out:
outputs = json.load(out)["outputs"]
gridnetwork_ip = outputs[gridnetwork_name + "-ip"]["value"]
pygrid_network_address = "=http://" + gridnetwork_ip if gridnetwork_ip else ""
host = "curl ifconfig.co" if gridnetwork_ip else "hostname -I"
image = terrascript.data.google_compute_image(
name + "container-optimized-os",
family="cos-81-lts",
project="cos-cloud",
)
self.config += image
# HOST environment variable is set to internal IP address
node = terrascript.resource.google_compute_instance(
name,
name=name,
machine_type=machine_type,
zone=zone,
boot_disk={
"initialize_params": {
"image": "${" + image.self_link + "}"
}
},
network_interface={
"network": "default",
"access_config": {}
},
metadata_startup_script=f"""
#!/bin/bash
sleep 30;
docker pull openmined/grid-node:production;
docker run \
-e NODE_ID={name} \
-e HOST = "$({host})" \
-e PORT=80 \
-e NETWORK={pygrid_network_address} \
-e DATABASE_URL=sqlite:///databasenode.db \
--name gridnode \
-p 80:80 \
-d openmined/grid-node:production;""",
)
self.config += node
with open("main.tf.json", "w") as main_config:
json.dump(self.config, main_config, indent=2, sort_keys=False)
if apply:
if IPython.get_ipython():
terraform_notebook.apply()
else:
terraform_script.apply()