def register(self, host=None):
self.verify_configuration()
# check if we are not already registered with the configured identity
resp = requests.get(f"{self.explorer_url}/users?name={self.tname}")
if resp.status_code == 404:
user = None
elif resp.status_code != 200:
resp.raise_for_status()
users = resp.json()
if not users:
user = None
else:
user = User.from_dict(users[0])
tid = None
if not user:
if not host:
try:
_, host = get_default_ip_config()
except Exception:
host = "localhost"
user = User()
user.name = self.tname
user.host = host
user.email = self.email
user.description = ""
user.pubkey = self.nacl.get_verify_key_hex()
resp = requests.post(f"{self.explorer_url}/users",
json=user.to_dict())
if resp.status_code == 409: # conflict
raise Input(
"A user with same name or email exists on TFGrid phonebook."
)
if resp.status_code != 201:
resp.raise_for_status()
tid = resp.json()["id"]
else:
if self.nacl.get_verify_key_hex() != user.pubkey:
raise Input(
"The verify key on your local system does not correspond with verify key on the TFGrid explorer"
)
tid = user.id
self._tid = tid
if self.tname not in self.admins:
self.admins.append(self.tname)
self.save()
return tid
def get(self, farm_id: int = None, farm_name: str = None) -> Farm:
"""
get the detail of a farm
:param farm_id: ID of the farm to retrieve
:type farm_id: int, optional
:param farm_name: Name of the farm to retrieve, if you use name the client loops over all farm and match on the name.
Be carefull using this in performance critical code path
:type farm_name: str, optional
:raises NotFound: if no farm with the specified ID or name if found
:raises Input: if farm_id and farm_name are None
:return: Farm object
:rtype: Farm
"""
if farm_name:
for farm in self.iter():
if farm.name == farm_name:
return farm
else:
raise NotFound(f"Could not find farm with name {farm_name}")
elif not farm_id:
raise Input("farms.get requires at least farm_id or farm_name")
resp = self._session.get(f"{self._url}/{farm_id}")
return Farm.from_dict(resp.json())
def sub_domain(self, gateway_id: str, domain: str, ips: List[str],
pool_id: int) -> GatewaySubdomain:
"""create a sub-domain workload object
Args:
gateway_id(str): the ID of the gateway where the create the sub-domain
domain(str): sub-domain to create
ips(List[str]): list of target IP pointed by the sub-domain
pool_id(int): capacity pool ID
Returns:
Subdomain: Subdomain
"""
for ip in ips:
if not _is_valid_ip(ip):
raise Input(f"{ip} is not valid IP address")
domain = self.correct_domain(domain)
sb = GatewaySubdomain()
sb.info.node_id = gateway_id
sb.domain = domain
sb.ips = ips
sb.info.workload_type = WorkloadType.Subdomain
sb.info.pool_id = pool_id
return sb
def _next_action(next_action):
if next_action:
if isinstance(next_action, str):
next_action = getattr(NextAction, next_action.upper()).value
if not isinstance(next_action, int):
raise Input("next_action should be of type int")
return next_action
def delete_access(self, network: Network, node_id: str,
iprange: str) -> Network:
"""remove a peer that was added using add_access method
use this is you want to revoke the access to the network from someone
Args:
network(Network): network object
node_id(str): ID of the node to use as entrypoint to the network
ip_range(str): IP range to allocate for this peer
Returns:
Network: Network
"""
node_workloads = {}
node_ranges = set()
for net_workload in network.network_resources:
node_workloads[net_workload.info.node_id] = net_workload
node_ranges.add(net_workload.iprange)
if iprange in node_ranges:
raise Input("Can't delete zos node peer")
access_workload = node_workloads.get(node_id)
if not access_workload:
raise Input(f"Node {node_id} is not part of network")
# remove peer from access node
new_peers = []
for peer in access_workload.peers:
if peer.iprange != iprange:
new_peers.append(peer)
access_workload.peers = new_peers
# remove peer from allowed ips on all nodes
access_node_range = node_workloads[node_id]
routing_range = wg_routing_ip(iprange)
for network_resource in node_workloads.values():
for peer in network_resource.peers:
if peer.iprange != access_node_range:
if iprange in peer.allowed_iprange:
peer.allowed_iprange.remove(iprange)
if routing_range in peer.allowed_iprange:
peer.allowed_iprange.remove(routing_range)
return network
def add_master(
self,
node_id: str,
network_name: str,
cluster_secret: str,
ip_address: str,
size: int,
ssh_keys: List[str],
pool_id: int,
public_ip_wid: int = 0,
disable_default_ingress=True,
datastore_endpoint="",
) -> K8s:
"""create a kubernetes marster workload object
Args:
node_id(str): node ID on which to deploy the k8s master
network_name(str): name of the network to use
cluster_secret(str): secret of the cluster. all the member of a same cluster must share the same secret
ip_address(str): ip address of the k8s master
size(int): size of the VM.
ssh_keys(List[str]): list of public SSH key to authorize in the VM
pool_id(int): capacity pool ID
Returns:
K8s: K8s
Raises:
Input: if size is not supported
"""
if size not in range(1, 19):
raise Input(f"VM size {size} is not supported")
master = K8s()
master.info.node_id = node_id
master.info.workload_type = WorkloadType.Kubernetes
master.info.pool_id = pool_id
node = self._nodes.get(node_id)
master.cluster_secret = encrypt_for_node(self._identity,
node.public_key_hex,
cluster_secret).decode()
master.network_id = network_name
master.ipaddress = ip_address
master.size = size
if not isinstance(ssh_keys, list):
ssh_keys = [ssh_keys]
master.ssh_keys = ssh_keys
master.public_ip = public_ip_wid
master.disable_default_ingress = disable_default_ingress
master.datastore_endpoint = datastore_endpoint
return master
def add_node(self,
network: Network,
node_id: str,
ip_range: str,
pool_id: int,
wg_port: int = None):
"""add a node into the network
Args:
network(Network): network object
node_id(str): ID of the node to add to the network
ip_range(str): IP range (cidr) to assign to the node inside the network. The network mask must be a /24
pool_id(int): the capacity pool ID
wg_port(int, optional
Raises:
Input: if mask of ip_range is not /24
Input: If specified access node not part of the network
Input: If access node point doesn't have a public endpoint
Input: If access node point doesn't have pubic endpoint of requested type): wireguar port of the wireguard endpoint. If None it will be picked automatically, defaults to None
Returns:
"""
node = self._nodes.get(node_id)
if netaddr.IPNetwork(ip_range).prefixlen != 24:
raise Input("ip_range should have a netmask of /24, not /%d",
ip_range.prefixlen)
if wg_port is None:
wg_port = _find_free_wg_port(node)
_, wg_private_encrypted, wg_public = generate_zos_keys(
node.public_key_hex)
nr = NetworkResource()
nr.info.pool_id = pool_id
nr.info.workload_type = WorkloadType.Network_resource
nr.network_iprange = network.iprange
nr.iprange = ip_range
nr.info.node_id = node_id
nr.wireguard_listen_port = wg_port
nr.wireguard_public_key = wg_public
nr.wireguard_private_key_encrypted = wg_private_encrypted
nr.name = network.name
network.network_resources.append(nr)
try:
self._load_network(network)
generate_peers(network)
finally:
self._cleanup_network(network)
def create(self, ip_range: str, network_name: str = None) -> Network:
"""create a new network
Args:
ip_range(str): IP range (cidr) of the full network. The network mask must be /16
network_name(str, optional): name of the network. If None, a random name will be generated, defaults to None
Returns:
Network: Network
Raises:
Input: if ip_range not a private range (RFC 1918)
Input: if network mask of ip_range is not /16
"""
network = netaddr.IPNetwork(ip_range)
if not is_private(network):
raise Input("ip_range must be a private network range (RFC 1918)")
if network.prefixlen != 16:
raise Input("network mask of ip range must be a /16")
network = Network(network_name, ip_range)
return network
def verify_configuration(self):
"""
Verifies passed arguments to constructor
Raises: NotFound incase tid is passed but does not exists on the explorer
Raises: Input: when params are missing
"""
if not self.words:
raise Input("Words are mandotory for an indentity")
if self._tid != -1:
resp = requests.get(f"{self.explorer_url}/users/{self._tid}")
resp.raise_for_status()
user = User.from_dict(resp.json())
if self.tname and self.tname != user.name:
raise Input("Name of user does not match name in explorer")
self.tname = user.name
if self.nacl.get_verify_key_hex() != user.pubkey:
raise Input(
"The verify key on your local system does not correspond with verify key on the TFGrid explorer"
)
else:
for key in ["email", "tname"]:
if not getattr(self, key):
raise Value(f"Threebot {key} not configured")
def add_master(
self,
node_id: str,
network_name: str,
cluster_secret: str,
ip_address: str,
size: int,
ssh_keys: List[str],
pool_id: int,
) -> K8s:
"""create a kubernetes marster workload object
Args:
node_id(str): node ID on which to deploy the k8s master
network_name(str): name of the network to use
cluster_secret(str): secret of the cluster. all the member of a same cluster must share the same secret
ip_address(str): ip address of the k8s master
size(int): size of the VM.
ssh_keys(List[str]): list of public SSH key to authorize in the VM
pool_id(int): capacity pool ID
Returns:
K8s: K8s
Raises:
Input: if size is not supported
"""
if size not in [1, 2]:
raise Input("size can only be 1 or 2")
master = K8s()
master.info.node_id = node_id
master.info.workload_type = WorkloadType.Kubernetes
master.info.pool_id = pool_id
node = self._nodes.get(node_id)
master.cluster_secret = encrypt_for_node(node.public_key_hex,
cluster_secret).decode()
master.network_id = network_name
master.ipaddress = ip_address
master.size = size
if not isinstance(ssh_keys, list):
ssh_keys = [ssh_keys]
master.ssh_keys = ssh_keys
return master
def run_cmd(self):
# get eab_kid and eab_hmac_key based on email or api_key_
if not self.email and not self.api_key_:
raise Input("email or api_key_ must be provided")
# set them to get the full run-cmd with correct arguments
if self.api_key_:
resp = j.tools.http.post(self.KEY_CREDENTIALS_URL, params={"access_key": self.api_key_})
else:
resp = j.tools.http.post(self.EMAIL_CREDENTIALS_URL, data={"email": self.email})
resp.raise_for_status()
data = resp.json()
self.eab_kid = data["eab_kid"]
self.eab_hmac_key = data["eab_hmac_key"]
return super().run_cmd
def is_access_node(node, version):
"""
Args:
node:
version:
Returns:
"""
if version not in [4, 6]:
raise Input("ip version can only be 4 or 6")
if node.public_config and node.public_config.master:
if version == 4:
return is_public_ip(node.public_config.ipv4, 4)
elif node.public_config.ipv6:
return is_public_ip(node.public_config.ipv6, 6)
return False
def configure_free_to_use(self, node_id: str, free: bool) -> bool:
"""
configure if the node capacity can be paid using FreeTFT
:param node_id: the node ID of the node to configure
:type node_id: str
:param free: if True, enable FreeTFT, if false disable it
:type free: bool
:return: True when the configuration was done successfully
:rtype: bool
"""
if not isinstance(free, bool):
raise Input("free must be a boolean")
data = {"free_to_use": free}
self._session.post(
f"{self._url}/{node_id}/configure_free", json=data,
)
return True
def create(
self,
node_id: str,
network_name: str,
name: str,
ip_address: str,
ssh_keys: List[str],
pool_id: int,
size: int,
public_ip_wid: int = 0,
) -> VirtualMachine:
"""create a virtual machine
Args:
node_id(str): node ID on which to deploy the vm
network_name(str): name of the network to use
ip_address(str): ip address of the vm
ssh_keys(List[str]): list of public SSH key to authorize in the VM
pool_id(int): capacity pool ID
name(str): the name of the vm image to deploy
public_ip_wid(int): The workload public ip
Returns:
vm: VirtualMachine
"""
if size not in range(1, 19):
raise Input(f"VM size {size} is not supported")
vm = VirtualMachine()
vm.info.node_id = node_id
vm.info.workload_type = WorkloadType.Virtual_Machine
vm.info.pool_id = pool_id
vm.network_id = network_name
vm.ipaddress = ip_address
if not isinstance(ssh_keys, list):
ssh_keys = [ssh_keys]
vm.ssh_keys = ssh_keys
vm.public_ip = public_ip_wid
vm.name = name
vm.size = size
return vm
def attach_existing(self, container: Container,
volume_id: Union[str, Volume], mount_point: str):
"""attach an existing volume to a container.
The volume must already exist on the node
Args:
container(Volume): container object returned from container.create_container function
volume_id(Union[str,volume]): the volume to attached to the container or its full ID
mount_point(str): path where to mount the volume in the container
"""
if isinstance(volume_id, Volume):
if not volume_id.id:
raise Input(
"volume needs to be deployed before it can be attached to a container"
)
volume_id = f"{volume_id.id}-1"
vol = ContainerMount()
vol.volume_id = volume_id
vol.mountpoint = mount_point
container.volumes.append(vol)
def check_url_reachable(url: str, timeout=5, verify=True):
"""Check that given url is reachable
Args:
url (str): url to test
timeout (int, optional): timeout of test. Defaults to 5.
verify (bool, optional): boolean indication to verify the servers TLS certificate or not.
Raises:
Input: raises if not correct url
Returns:
bool: True if the test succeeds, False otherwise
"""
try:
code = jumpscale.tools.http.get(url, timeout=timeout,
verify=verify).status_code
return code == 200
except jumpscale.tools.http.exceptions.MissingSchema:
raise Input("Please specify correct url with correct scheme")
except jumpscale.tools.http.exceptions.ConnectionError:
return False
def filter_public_ip(node, version):
"""
Args:
node:
version:
Returns:
"""
if version not in [4, 6]:
raise Input("ip version can only be 4 or 6")
if node.public_config and node.public_config.master:
if version == 4:
return is_public_ip(node.public_config.ipv4, 4)
elif node.public_config.ipv6:
return is_public_ip(node.public_config.ipv6, 6)
else:
for iface in node.ifaces:
for addr in iface.addrs:
if is_public_ip(addr, version):
return True
return False
def add_access(self,
network: Network,
node_id: str,
ip_range: str,
wg_public_key: str = None,
ipv4: bool = False) -> str:
"""add an external access to the network. use this function if you want to allow
a peer to your network that is not a 0-OS node like User laptop, external server,...
Args:
network(Network): network object
node_id(str): ID of the node to use as entrypoint to the network
ip_range(str): IP range to allocate for this peer
wg_public_key(str, optional): wireguard public key of the peer. If not specify a new key will be generated, defaults to None
ipv4(bool, optional): If True, use an IPv4 address as endpoint to connect to the network otherwise use IPv6, defaults to False
Returns:
str: the wireguard configuration to be used by the peer to connect to the network
"""
if netaddr.IPNetwork(ip_range).prefixlen != 24:
raise Input("ip_range should have a netmask of /24, not /%d",
ip_range.prefixlen)
try:
self._load_network(network)
access_point_nr = None
for nr in network.network_resources:
if node_id == nr.info.node_id:
access_point_nr = nr
if access_point_nr is None:
raise Input(
"can not add access through a node which is not in the network"
)
if len(access_point_nr.public_endpoints) == 0:
raise Input("access node must have at least 1 public endpoint")
endpoint = ""
wg_port = access_point_nr.wireguard_listen_port
for ep in access_point_nr.public_endpoints:
if ipv4 and ep.version == 4:
endpoint = f"{str(ep.ip)}:{wg_port}"
break
if not ipv4 and ep.version == 6:
ip = str(ep.ip)
endpoint = f"[{ip}]:{wg_port}"
break
if not endpoint:
raise Input(
"access node has no public endpoint of the requested type")
wg_private_key = ""
if not wg_public_key:
wg_private = public.PrivateKey.generate()
wg_public = wg_private.public_key
wg_private_key = wg_private.encode(Base64Encoder).decode()
wg_public_key = wg_public.encode(Base64Encoder).decode()
network.access_points.append(
AccessPoint(node_id=node_id,
subnet=ip_range,
wg_public_key=wg_public_key,
ip4=ipv4))
generate_peers(network)
finally:
self._cleanup_network(network)
return generate_wg_quick(wg_private_key, ip_range,
access_point_nr.wireguard_public_key,
network.iprange, endpoint)