class ReservationResult(Base):
category = fields.Enum(Category)
workload_id = fields.String(default="")
data_json = fields.Json()
signature = fields.Bytes()
state = fields.Enum(State)
message = fields.String(default="")
epoch = fields.DateTime()
class ZdbNamespace(Base):
id = fields.Integer()
node_id = fields.String(default="")
size = fields.Integer()
mode = fields.Enum(ZDBMode)
password = fields.String(default="")
disk_type = fields.Enum(DiskType)
public = fields.Boolean(default=False)
stats_aggregator = fields.List(fields.Object(Statsaggregator))
info = fields.Object(ReservationInfo)
class UserThreebot(Base):
# instance name is the f"threebot_{solution uuid}"
solution_uuid = fields.String()
identity_tid = fields.Integer()
name = fields.String()
owner_tname = fields.String(
) # owner's tname in ThreeFold Connect after cleaning
farm_name = fields.String()
state = fields.Enum(ThreebotState)
continent = fields.String()
explorer_url = fields.String()
threebot_container_wid = fields.Integer()
trc_container_wid = fields.Integer(
) # deprecated for embeding trc # FIXME: Remove
reverse_proxy_wid = fields.Integer(
) # deprecated for embeding trc # FIXME: Remove
subdomain_wid = fields.Integer()
secret_hash = fields.String()
proxy_wid = fields.Integer()
def verify_secret(self, secret):
if not self.secret_hash:
return True
return self.secret_hash == hashlib.md5(secret.encode()).hexdigest()
def hash_secret(self, secret):
self.secret_hash = hashlib.md5(secret.encode()).hexdigest()
class KubernetesNode(VDCHostBase):
role = fields.Enum(KubernetesRole)
public_ip = fields.IPRange()
_size = fields.Integer()
@property
def size(self):
return VDC_SIZE.K8SNodeFlavor(self._size)
@classmethod
def from_workload(cls, workload):
node = cls()
node.wid = workload.id
node.ip_address = workload.ipaddress
if workload.master_ips:
node.role = KubernetesRole.WORKER
else:
node.role = KubernetesRole.MASTER
node.node_id = workload.info.node_id
node.pool_id = workload.info.pool_id
if workload.public_ip:
zos = get_zos()
public_ip_workload = zos.workloads.get(workload.public_ip)
address = str(netaddr.IPNetwork(public_ip_workload.ipaddress).ip)
node.public_ip = address
node._size = (
VDC_SIZE.K8SNodeFlavor(workload.size).value
if workload.size in [size.value for size in K8S_SIZES]
else VDC_SIZE.K8SNodeFlavor.SMALL.value
)
return node
class TfgridSolution1(Base):
id = fields.Integer()
name = fields.String(default="")
solution_type = fields.Enum(SolutionType)
rid = fields.Integer()
form_info = fields.Typed(dict)
explorer = fields.String(default="")
class User(Base):
emails = fields.List(fields.String())
permissions = fields.List(fields.Object(Permission))
custom_config = fields.Typed(dict)
type = fields.Enum(UserType)
password = fields.Secret()
first_name = fields.String(default="")
last_name = fields.String(default="")
def get_full_name(self):
name = self.first_name
if self.last_name:
name += " " + self.last_name
return name
def get_unique_name(self):
return self.full_name.replace(" ", "") + ".user"
full_name = fields.String(compute=get_full_name)
unique_name = fields.String(compute=get_unique_name)
def get_my_greeter(self):
return Greeter(self.full_name)
my_greeter = fields.Typed(Greeter, stored=False, compute=get_my_greeter)
ahmed_greeter = fields.Typed(Greeter,
stored=False,
default=Greeter("ahmed"))
class ResourceUnitPrice(Base):
currency = fields.Enum(Currency)
cru = fields.Float()
mru = fields.Float()
hru = fields.Float()
sru = fields.Float()
nru = fields.Float()
class NodePublicIface(Base):
master = fields.String(default="")
type = fields.Enum(NicType)
ipv4 = fields.IPRange()
ipv6 = fields.IPRange()
gw4 = fields.IPRange()
gw6 = fields.IPRange()
version = fields.Integer()
class ReservationInfo(Base):
workload_id = fields.Integer()
node_id = fields.String()
pool_id = fields.Integer()
description = fields.String(default="")
reference = fields.String(default="")
customer_tid = fields.Integer()
customer_signature = fields.String()
next_action = fields.Enum(NextAction)
signatures_provision = fields.List(fields.Object(Signature))
signing_request_provision = fields.Object(SigningRequest)
signing_request_delete = fields.Object(SigningRequest)
signatures_farmer = fields.List(fields.Object(Signature))
signatures_delete = fields.List(fields.Object(Signature))
epoch = fields.DateTime(default=datetime.utcnow)
metadata = fields.String(default="")
result = fields.Object(ReservationResult)
workload_type = fields.Enum(WorkloadType)
class ZdbNamespace(Base):
id = fields.Integer()
node_id = fields.String(default="")
size = fields.Integer()
mode = fields.Enum(ZDBMode)
password = fields.String(default="")
disk_type = fields.Enum(DiskType)
public = fields.Boolean(default=False)
stats_aggregator = fields.List(fields.Object(Statsaggregator))
info = fields.Object(ReservationInfo)
def resource_units(self):
resource_units = ResourceUnitAmount()
if self.disk_type == DiskType.HDD:
resource_units.hru += self.size
elif self.disk_type == DiskType.SSD:
resource_units.sru += self.size
return resource_units
class APIKey(Base):
key = fields.String(default=lambda: uuid4().hex)
role = fields.Enum(UserRole)
created_at = fields.Float(default=lambda: j.data.time.utcnow().timestamp)
def to_dict(self):
d = super().to_dict()
d["name"] = self.instance_name
return d
class Location(Base):
path_url = fields.String(default="/")
force_https = fields.Boolean(default=False)
path_location = fields.String(default="/")
index = fields.String(default="index.html")
scheme = fields.String(default="http")
host = fields.String(default="127.0.0.1")
port = fields.Integer()
path_dest = fields.String(default="/")
spa = fields.Boolean(default=False)
websocket = fields.Boolean(default=False)
location_type = fields.Enum(LocationType)
is_auth = fields.Boolean(default=False)
is_admin = fields.Boolean(default=False)
package_name = fields.String()
is_package_authorized = fields.Boolean(default=False)
custom_config = fields.String(default=None)
proxy_buffering = fields.Enum(ProxyBuffering)
proxy_buffers = fields.String()
proxy_buffer_size = fields.String()
@property
def cfg_dir(self):
return j.sals.fs.join_paths(self.parent.cfg_dir, "locations")
@property
def cfg_file(self):
return j.sals.fs.join_paths(self.cfg_dir, f"{self.instance_name}.conf")
def get_config(self):
return render_config_template(
"location",
base_dir=j.core.dirs.BASEDIR,
location=self,
threebot_connect=j.core.config.get_config().get("threebot_connect", True),
https_port=PORTS.HTTPS,
)
def configure(self):
j.sals.fs.mkdir(self.cfg_dir)
j.sals.fs.write_file(self.cfg_file, self.get_config())
class Reservation(Base):
id = fields.Integer()
json = fields.String(default="")
data_reservation = fields.Object(ReservationData)
customer_tid = fields.Integer()
customer_signature = fields.String(default="")
next_action = fields.Enum(NextAction)
signatures_provision = fields.List(fields.Object(Signature))
signatures_farmer = fields.List(fields.Object(Signature))
signatures_delete = fields.List(fields.Object(Signature))
epoch = fields.DateTime(default=datetime.utcnow)
metadata = fields.String(default="")
results = fields.List(fields.Object(ReservationResult))
class Volume(Base):
id = fields.Integer()
size = fields.Integer()
type = fields.Enum(DiskType)
stats_aggregator = fields.List(fields.Object(Statsaggregator))
info = fields.Object(ReservationInfo)
def resource_units(self):
resource_units = ResourceUnitAmount()
if self.type == DiskType.HDD:
resource_units.hru += self.size
elif self.type == DiskType.SSD:
resource_units.sru += self.size
return resource_units
class VDCWallet(Base):
vdc_uuid = fields.String()
wallet_secret = fields.String()
wallet_network = fields.Enum(StellarNetwork)
def _init_wallet(self, secret=None):
wallet = j.clients.stellar.new(self.instance_name, secret=secret)
if not secret:
wallet.activate_through_activation_wallet()
wallet.save()
self.wallet_secret = wallet.secret
@property
def stellar_wallet(self):
if not j.clients.stellar.find(
self.instance_name) and self.wallet_secret:
self._init_wallet(self.wallet_secret)
return j.clients.stellar.get(self.instance_name)
class Location(Base):
name = fields.String()
path_url = fields.String()
is_auth = fields.Boolean(default=False)
force_https = fields.Boolean(default=False)
path_location = fields.String()
index = fields.String()
ipaddr_dest = fields.String()
port_dest = fields.Integer()
path_dest = fields.String()
location_type = fields.Enum(LocationType)
scheme = fields.String()
@property
def path_cfg_dir(self):
return f"{self.parent.path_cfg_dir}/{self.parent.instance_name}_locations"
@property
def path_cfg(self):
return f"{self.path_cfg_dir}/{self.instance_name}.conf"
@property
def path_web(self):
return self.parent.path_web
def write_config(self, content=""):
if not content:
content = render_config_template(f"location_{self.location_type}",
obj=self)
j.sals.fs.write_file(self.path_cfg, content)
def configure(self):
"""Config is a server config file of nginx (in text format)
"""
j.sals.fs.mkdir(self.path_cfg_dir)
if self.location_type.value in [
LocationType.STATIC.value, LocationType.SPA.value
]:
if not self.path_location.endswith("/"):
self.path_location += "/"
self.write_config(self.config)
class Volume(Base):
id = fields.Integer()
size = fields.Integer()
type = fields.Enum(DiskType)
stats_aggregator = fields.List(fields.Object(Statsaggregator))
info = fields.Object(ReservationInfo)
class Car(Base):
color = fields.Enum(Colors)
class ThreebotServer(Base):
_package_manager = fields.Factory(PackageManager)
domain = fields.String()
email = fields.String()
acme_server_type = fields.Enum(AcmeServer)
acme_server_url = fields.URL()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._rack = None
self._gedis = None
self._db = None
self._gedis_http = None
self._services = None
self._packages = None
self._started = False
self._nginx = None
self._redis = None
self.rack.add(GEDIS, self.gedis)
self.rack.add(GEDIS_HTTP, self.gedis_http.gevent_server)
self.rack.add(SERVICE_MANAGER, self.services)
def is_running(self):
nginx_running = self.nginx.is_running()
redis_running = self.redis.cmd.is_running(
) or j.sals.nettools.wait_connection_test("127.0.0.1", 6379, timeout=1)
gedis_running = j.sals.nettools.wait_connection_test("127.0.0.1",
16000,
timeout=1)
return nginx_running and redis_running and gedis_running
@property
def started(self):
return self._started
@property
def nginx(self):
if self._nginx is None:
self._nginx = j.tools.nginx.get("default")
return self._nginx
@property
def redis(self):
if self._redis is None:
self._redis = j.tools.redis.get("default")
return self._redis
@property
def db(self):
if self._db is None:
self._db = j.core.db
return self._db
@property
def rack(self):
if self._rack is None:
self._rack = j.servers.rack
return self._rack
@property
def gedis(self):
if self._gedis is None:
self._gedis = j.servers.gedis.get("threebot")
return self._gedis
@property
def gedis_http(self):
if self._gedis_http is None:
self._gedis_http = j.servers.gedis_http.get("threebot")
return self._gedis_http
@property
def services(self):
if self._services is None:
self._services = j.tools.servicemanager.get("threebot")
return self._services
@property
def chatbot(self):
return self.gedis._loaded_actors.get("chatflows_chatbot")
@property
def packages(self):
if self._packages is None:
self._packages = self._package_manager.get(self.instance_name)
return self._packages
def check_dependencies(self):
install_msg = "Visit https://github.com/threefoldtech/js-sdk/blob/development/docs/wiki/quick_start.md for installation guide"
if not self.nginx.installed:
raise j.exceptions.NotFound(
f"nginx is not installed.\n{install_msg}")
ret = shutil.which("certbot")
if not ret:
raise j.exceptions.NotFound(
f"certbot is not installed.\n{install_msg}")
rc, out, err = j.sals.process.execute("certbot plugins")
if "* nginx" not in out:
raise j.exceptions.NotFound(
f"python-certbot-nginx is not installed.\n{install_msg}")
if not self.redis.installed:
raise j.exceptions.NotFound(
f"redis is not installed.\n{install_msg}")
ret = shutil.which("tmux")
if not ret:
raise j.exceptions.NotFound(
f"tmux is not installed.\n{install_msg}")
ret = shutil.which("git")
if not ret:
raise j.exceptions.NotFound(
f"git is not installed.\n{install_msg}")
def start(self, wait: bool = False):
# start default servers in the rack
# handle signals
for signal_type in (signal.SIGTERM, signal.SIGINT, signal.SIGKILL):
gevent.signal(signal_type, self.stop)
# mark app as started
if self.is_running():
return
self.check_dependencies()
self.redis.start()
self.nginx.start()
self.rack.start()
j.logger.register(f"threebot_{self.instance_name}")
# add default packages
for package_name in DEFAULT_PACKAGES:
j.logger.info(f"Configuring package {package_name}")
try:
package = self.packages.get(package_name)
self.packages.install(package)
except Exception as e:
self.stop()
raise j.core.exceptions.Runtime(
f"Error happened during getting or installing {package_name} package, the detailed error is {str(e)}"
) from e
# install all package
self.packages._install_all()
j.logger.info("Reloading nginx")
self.nginx.reload()
# mark server as started
self._started = True
j.logger.info(
f"Threebot is running at http://localhost:{PORTS.HTTP} and https://localhost:{PORTS.HTTPS}"
)
self.rack.start(wait=wait) # to keep the server running
def stop(self):
server_packages = self.packages.list_all()
for package_name in server_packages:
package = self.packages.get(package_name)
package.stop()
self.nginx.stop()
# mark app as stopped, do this before stopping redis
j.logger.unregister()
self.redis.stop()
self.rack.stop()
self._started = False
class Wallet(Base):
id = fields.String()
address = fields.String()
balance = fields.Float()
currency = fields.Enum(Currency)
tags = fields.List(fields.String())
class StartupCmd(Base):
start_cmd = fields.String()
ports = fields.List(fields.Integer())
executor = fields.Enum(Executor)
check_cmd = fields.String()
path = fields.String(default=j.core.dirs.TMPDIR)
stop_cmd = fields.String()
env = fields.Typed(dict, default={})
timeout = fields.Integer(default=60)
process_strings = fields.List(fields.String())
process_strings_regex = fields.List(fields.String())
def __init__(self):
super().__init__()
self._process = None
self._pid = None
self._cmd_path = None
self.__tmux_window = None
def reset(self):
self._process = None
self._pid = None
@property
def pid(self):
if not self._pid:
pids = j.sals.process.get_pids(f"startupcmd_{self.instance_name}")
if pids:
self._pid = pids[0]
return self._pid
@property
def cmd_path(self):
if not self._cmd_path:
self._cmd_path = j.sals.fs.join_paths(j.core.dirs.VARDIR, "cmds",
f"{self.instance_name}.sh")
j.sals.fs.mkdirs(j.sals.fs.dirname(self._cmd_path))
return self._cmd_path
@pid.setter
def pid(self, pid):
self._pid = pid
@property
def process(self):
if not self._process:
if self.pid:
self._process = j.sals.process.get_process_object(self.pid,
die=False)
if not self._process:
self.pid = None
else:
processes = self._get_processes_by_port_or_filter()
if len(processes) == 1:
self._process = processes[0]
return self._process
@property
def _tmux_window(self):
if self.executor == Executor.TMUX:
if self.__tmux_window is None:
self.__tmux_window = j.core.executors.tmux.get_js_window(
self.instance_name)
return self.__tmux_window
def _get_processes_by_port_or_filter(self):
"""Uses object properties to find the corresponding process(es)
Returns:
list: All processes that matched
"""
pids_done = []
result = []
def _add_to_result(process):
if process and process.pid not in pids_done:
result.append(process)
pids_done.append(process.pid)
for port in self.ports:
try:
process = j.sals.process.get_process_by_port(port)
except Exception:
continue
_add_to_result(process)
for process_string in self.process_strings:
for pid in j.sals.process.get_filtered_pids(process_string):
process = j.sals.process.get_process_object(pid, die=False)
_add_to_result(process)
for pid in j.sals.process.get_pids_filtered_by_regex(
self.process_strings_regex):
process = j.sals.process.get_process_object(pid, die=False)
_add_to_result(process)
# We return all processes which match
return result
def _kill_processes_by_port_or_filter(self):
"""Kills processes that matches object properties
"""
processes = self._get_processes_by_port_or_filter()
self._kill_processes(processes)
def _kill_processes(self, processes):
"""Kill processes
Args:
processes (list): List of processes
"""
for process in processes:
try:
process.kill()
except NoSuchProcess:
pass # already killed
def _soft_kill(self):
"""Kills the poocess using `stop_cmd`
Returns:
bool: True if was killed
"""
if self.stop_cmd:
cmd = j.tools.jinja2.render_template(template_text=self.stop_cmd,
args=self._get_data())
exit_code, _, _ = j.sals.process.execute(cmd, die=False)
self.reset()
return exit_code == 0
return False
def _hard_kill(self):
"""Force Kills the process
"""
if self.process:
self._kill_processes([self.process])
self.reset()
self._kill_processes_by_port_or_filter()
if self.executor == Executor.TMUX:
self._tmux_window.kill_window()
self.__tmux_window = None
def stop(self, force=True, wait_for_stop=True, die=True, timeout=None):
"""Stops the running command
Args:
force (bool, optional): If True will force kill the process. Defaults to True.
wait_for_stop (bool, optional): If True will wait until process is stopped. Defaults to True.
die (bool, optional): If True will raise if timeout is exceeded for stop. Defaults to True.
timeout (int, optional): Timeout for stop wait.If not set will use `timeout` property. Defaults to None.
"""
timeout = timeout or self.timeout
if self.is_running():
self._soft_kill()
if force:
self._hard_kill()
if wait_for_stop:
self.wait_for_stop(die=die, timeout=timeout)
j.sals.process.execute(f"rm {self.cmd_path}", die=False)
def is_running(self):
"""Checks if startup cmd is running. Will use `check_cmd` property if defined or check based on objet properties
Returns:
bool: True if it is running
"""
if self.check_cmd:
exit_code, _, _ = j.sals.process.execute(self.check_cmd, die=False)
return exit_code == 0
self.reset()
if self.process:
return self.process.is_running()
return self._get_processes_by_port_or_filter() != []
def _wait(self, for_running, die, timeout):
"""Wait for either start or stop to finishes
Args:
for_running (bool): Whether to check if it is running or stopped.
die (bool, optional): If True will raise if timeout is exceeded for stop. Defaults to True.
timeout (int, optional): Timeout for wait operation. Defaults to None.
Raises:
j.exceptions.Timeout: If timeout is exceeded.
"""
end = j.data.time.now().timestamp + timeout
while j.data.time.now().timestamp < end:
if self.is_running() == for_running:
break
time.sleep(0.05)
else:
if die:
raise j.exceptions.Timeout(
f"Wait operation exceeded timeout: {timeout}")
def wait_for_stop(self, die=True, timeout=10):
"""Wait for stop to finishes
Args:
die (bool, optional): If True will raise if timeout is exceeded for stop. Defaults to True.
timeout (int, optional): Timeout for wait operation. Defaults to None.
Raises:
j.exceptions.Timeout: If timeout is exceeded.
"""
self._wait(False, die, timeout)
def wait_for_running(self, die=True, timeout=10):
"""Wait for start to finishes
Args:
die (bool, optional): If True will raise if timeout is exceeded for stop. Defaults to True.
timeout (int, optional): Timeout for wait operation. Defaults to None.
Raises:
j.exceptions.Timeout: If timeout is exceeded.
"""
self._wait(True, die, timeout)
def start(self):
"""Starts the process
"""
if self.is_running():
return
if not self.start_cmd:
raise j.exceptions.Value("please make sure start_cmd has been set")
if "\n" in self.start_cmd.strip():
command = self.start_cmd
else:
template_script = """
set +ex
{% for key,val in env.items() %}
export {{key}}='{{val}}'
{% endfor %}
mkdir -p {{path}}
cd {{path}}
bash -c \"exec -a startupcmd_{{name}} {{start_cmd}}\"
"""
script = j.tools.jinja2.render_template(
template_text=template_script,
env=self.env,
path=self.path,
start_cmd=self.start_cmd,
name=self.instance_name,
)
j.sals.fs.write_file(self.cmd_path, script)
j.sals.fs.chmod(self.cmd_path, 0o770)
command = f"sh {self.cmd_path}"
if self.executor == Executor.FOREGROUND:
j.sals.process.execute(command)
elif self.executor == Executor.TMUX:
self._tmux_window.attached_pane.send_keys(command)
self.wait_for_running(die=True, timeout=self.timeout)
class Website(Base):
domain = fields.String()
ssl = fields.Boolean()
port = fields.Integer(default=PORTS.HTTP)
locations = fields.Factory(Location, stored=False)
includes = fields.List(fields.String())
selfsigned = fields.Boolean(default=True)
# keep it as letsencryptemail for compatibility
letsencryptemail = fields.String()
acme_server_type = fields.Enum(AcmeServer)
acme_server_url = fields.URL()
# in case of using existing key/certificate
key_path = fields.String()
cert_path = fields.String()
fullchain_path = fields.String()
@property
def certbot(self):
kwargs = dict(
domain=self.domain,
email=self.letsencryptemail,
server=self.acme_server_url,
nginx_server_root=self.parent.cfg_dir,
key_path=self.key_path,
cert_path=self.cert_path,
fullchain_path=self.fullchain_path,
)
if self.acme_server_type == AcmeServer.LETSENCRYPT:
certbot_type = LetsencryptCertbot
elif self.acme_server_type == AcmeServer.ZEROSSL:
certbot_type = ZerosslCertbot
else:
certbot_type = CustomCertbot
return certbot_type(**kwargs)
@property
def cfg_dir(self):
return j.sals.fs.join_paths(self.parent.cfg_dir, self.instance_name)
@property
def cfg_file(self):
return j.sals.fs.join_paths(self.cfg_dir, "server.conf")
@property
def include_paths(self):
paths = []
for include in self.includes:
## TODO validate location name and include
website_name, location_name = include.split(".", 1)
website = self.parent.websites.find(website_name)
if not website:
continue
paths.append(j.sals.fs.join_paths(website.cfg_dir, "locations", location_name))
return paths
def get_locations(self):
for location in self.locations.list_all():
yield self.locations.get(location)
def get_proxy_location(self, name):
location = self.locations.get(name)
location.location_type = LocationType.PROXY
return location
def get_custom_location(self, name):
location = self.locations.get(name)
location.location_type = LocationType.CUSTOM
return location
def get_static_location(self, name):
location = self.locations.get(name)
location.location_type = LocationType.STATIC
return location
def get_config(self):
return render_config_template("website", base_dir=j.core.dirs.BASEDIR, website=self)
def generate_certificates(self, retries=6):
if self.domain:
if self.key_path and self.cert_path and self.fullchain_path:
# only use install command if an existing key and certificate were set
self.install_certifcate()
else:
self.obtain_and_install_certifcate(retries=retries)
def install_certifcate(self):
"""Construct and Execute install certificate command
Alternative to certbot install
"""
cmd = self.certbot.install_cmd
j.logger.debug(f"Execute: {' '.join(cmd)}")
rc, out, err = j.sals.process.execute(cmd)
if rc > 0:
j.logger.error(f"Installing certificate failed {out}\n{err}")
else:
j.logger.info(f"Certificate installed successfully {out}")
def obtain_and_install_certifcate(self, retries=6):
"""Construct and Execute run certificate command,This will issue a new certificate managed by Certbot
Alternative to certbot run
Args:
retries (int, optional): Number of retries Certbot will try to install the certificate if failed. Defaults to 6.
"""
cmd = self.certbot.run_cmd
j.logger.debug(f"Execute: {' '.join(cmd)}")
for _ in range(retries):
rc, out, err = j.sals.process.execute(cmd)
if rc > 0:
j.logger.error(f"Generating certificate failed {out}\n{err}")
else:
j.logger.error(f"Certificate Generated successfully {out}")
break
def generate_self_signed_certificates(self):
keypempath = f"{self.parent.cfg_dir}/key.pem"
certpempath = f"{self.parent.cfg_dir}/cert.pem"
if j.sals.process.is_installed("mkcert"):
res = j.sals.process.execute(
f"mkcert -key-file {keypempath} -cert-file {certpempath} localhost *.localhost 127.0.0.1 ::1"
)
if res[0] != 0:
raise j.exceptions.JSException(f"Failed to generate self-signed certificate (using mkcert).{res}")
else:
if j.sals.fs.exists(f"{keypempath}") and j.sals.fs.exists(f"{certpempath}"):
return
res = j.sals.process.execute(
f"openssl req -nodes -x509 -newkey rsa:4096 -keyout {keypempath} -out {certpempath} -days 365 -subj '/CN=localhost'"
)
if res[0] != 0:
raise j.exceptions.JSException(f"Failed to generate self-signed certificate (using openssl).{res}")
def configure(self, generate_certificates=True):
j.sals.fs.mkdir(self.cfg_dir)
needed_dirs = ("body", "client-body", "fastcgi", "proxy", "scgi", "uwsgi")
for d in needed_dirs:
j.sals.fs.mkdir(j.sals.fs.join_paths(self.cfg_dir, d))
for location in self.get_locations():
location.configure()
j.sals.fs.write_file(self.cfg_file, self.get_config())
if self.ssl:
self.generate_self_signed_certificates()
if generate_certificates and self.ssl:
self.generate_certificates()
def clean(self):
j.sals.fs.rmtree(self.cfg_dir)
class UserVDC(Base):
vdc_name = fields.String()
owner_tname = fields.String()
solution_uuid = fields.String(default=lambda: uuid.uuid4().hex)
identity_tid = fields.Integer()
s3 = fields.Object(S3)
vmachines = fields.List(fields.Object(VMachine))
kubernetes = fields.List(fields.Object(KubernetesNode))
etcd = fields.List(fields.Object(ETCDNode))
threebot = fields.Object(VDCThreebot)
created = fields.DateTime(default=datetime.datetime.utcnow)
expiration = fields.Float(default=lambda: j.data.time.utcnow().timestamp + 30 * 24 * 60 * 60)
last_updated = fields.DateTime(default=datetime.datetime.utcnow)
is_blocked = fields.Boolean(default=False) # grace period action is applied
explorer_url = fields.String(default=lambda: j.core.identity.me.explorer_url)
_flavor = fields.String()
state = fields.Enum(VDCSTATE)
__lock = BoundedSemaphore(1)
transaction_hashes = []
@property
def flavor(self):
d = self.to_dict()
oldflavor = d.get("flavor", "silver")
if not self._flavor:
flavors = {"silver": "silver", "gold": "gold", "platinum": "platinum", "diamond": "diamond"}
self._flavor = d.get(flavors[oldflavor])
self.save()
# TODO: should we do a save here?
return VDC_SIZE.VDCFlavor(self._flavor)
else:
return VDC_SIZE.VDCFlavor(self._flavor)
def to_dict(self):
d = super().to_dict()
d["flavor"] = self._flavor
for node in d["kubernetes"]:
node["size"] = node["_size"]
return d
def is_empty(self, load_info=True):
if load_info:
self.load_info()
if any([self.kubernetes, self.threebot.wid, self.threebot.domain, self.s3.minio.wid, self.s3.zdbs]):
return False
self.state = VDCSTATE.EMPTY
self.save()
return True
def has_minimal_components(self):
if all([self.kubernetes, self.threebot.wid, self.threebot.domain]):
return True
return False
@property
def expiration_date(self):
expiration = self.calculate_expiration_value()
return j.data.time.get(expiration).datetime
@property
def prepaid_wallet(self):
wallet_name = f"prepaid_wallet_{self.solution_uuid}"
wallet = j.clients.stellar.find(wallet_name)
if not wallet:
vdc_wallet = VDC_WALLET_FACTORY.find(wallet_name)
if not vdc_wallet:
vdc_wallet = VDC_WALLET_FACTORY.new(wallet_name)
vdc_wallet.save()
wallet = vdc_wallet.stellar_wallet
return wallet
@property
def provision_wallet(self):
wallet_name = f"provision_wallet_{self.solution_uuid}"
wallet = j.clients.stellar.find(wallet_name)
if not wallet:
vdc_wallet = VDC_WALLET_FACTORY.find(wallet_name)
if not vdc_wallet:
vdc_wallet = VDC_WALLET_FACTORY.new(wallet_name)
vdc_wallet.save()
wallet = vdc_wallet.stellar_wallet
return wallet
@property
def vdc_workloads(self):
workloads = []
workloads += self.kubernetes
workloads += self.s3.zdbs
workloads += self.vmachines
if self.threebot.wid:
workloads.append(self.threebot)
if self.s3.minio.wid:
workloads.append(self.s3.minio)
return workloads
@property
def active_pools(self):
my_pool_ids = [w.pool_id for w in self.vdc_workloads]
explorer = j.core.identity.me.explorer
active_pools = [p for p in explorer.pools.list(customer_tid=self.identity_tid) if p.pool_id in my_pool_ids]
return active_pools
def get_deployer(
self,
password=None,
identity=None,
bot=None,
proxy_farm_name=None,
deployment_logs=False,
ssh_key_path=None,
restore=False,
network_farm=None,
compute_farm=None,
):
proxy_farm_name = proxy_farm_name or random.choice(PROXY_FARMS.get())
if not password and not identity:
identity = self._get_identity()
return VDCDeployer(
vdc_instance=self,
password=password,
bot=bot,
proxy_farm_name=proxy_farm_name,
identity=identity,
deployment_logs=deployment_logs,
ssh_key_path=ssh_key_path,
restore=restore,
network_farm=network_farm,
compute_farm=compute_farm,
)
def get_password(self):
identity = self._get_identity(default=False)
if not identity:
j.logger.error("Couldn't find identity")
return
password_hash = j.data.encryption.mnemonic_to_key(identity.words)
return password_hash.decode()
def validate_password(self, password):
password = j.data.hash.md5(password)
vdc_password = self.get_password()
if not vdc_password:
# identity was not generated for this vdc instance
return True
if password == vdc_password:
return True
return False
def _get_identity(self, default=True):
instance_name = f"vdc_ident_{self.solution_uuid}"
identity = None
if j.core.identity.find(instance_name):
identity = j.core.identity.find(instance_name)
elif default:
identity = j.core.identity.me
return identity
def get_zdb_monitor(self):
return ZDBMonitor(self)
def get_kubernetes_monitor(self):
return KubernetesMonitor(self)
def get_snapshot_manager(self, snapshots_dir=None):
return SnapshotManager(self, snapshots_dir)
def get_quantumstorage_manager(self, ip_version=4):
return QuantumStorage(self, ip_version)
def load_info(self, load_proxy=False):
kubernetes, s3, vmachines, etcd, threebot = self.get_vdc_workloads(load_proxy=load_proxy)
self.__lock.acquire()
try:
self.kubernetes = kubernetes
self.etcd = etcd
self.s3 = s3
self.vmachines = vmachines
self.threebot = threebot
finally:
self.__lock.release()
def _build_zdb_proxies(self, s3):
proxies = self._list_socat_proxies()
for zdb in s3.zdbs:
zdb_proxies = proxies[zdb.ip_address]
if not zdb_proxies:
continue
proxy = zdb_proxies[0]
zdb.proxy_address = f"{proxy['ip_address']}:{proxy['listen_port']}"
def get_public_ip(self):
if not self.kubernetes:
self.load_info()
public_ip = None
for node in self.kubernetes:
if node.public_ip != "::/128":
public_ip = node.public_ip
break
return public_ip
def _list_socat_proxies(self, public_ip=None):
public_ip = public_ip or self.get_public_ip()
if not public_ip:
raise j.exceptions.Runtime(f"Couldn't get a public ip for vdc: {self.vdc_name}")
ssh_client = self.get_ssh_client("socat_list", public_ip, "rancher")
result = defaultdict(list)
rc, out, _ = ssh_client.sshclient.run(f"sudo ps -ef | grep -v grep | grep socat", warn=True)
if rc != 0:
return result
for line in out.splitlines():
# root 6659 1 0 Feb19 ? 00:00:00 /var/lib/rancher/k3s/data/current/bin/socat tcp-listen:9900,reuseaddr,fork tcp:[2a02:1802:5e:0:c46:cff:fe32:39ae]:9900
splits = line.split("tcp-listen:")
if len(splits) != 2:
continue
splits = splits[1].split(",")
if len(splits) < 2:
continue
listen_port = splits[0]
splits = line.split("tcp:")
if len(splits) != 2:
continue
proxy_address = splits[1]
splits = proxy_address.split(":")
if len(splits) < 2:
continue
port = splits[-1]
ip_address = ":".join(splits[:-1])
if ip_address[0] == "[" and ip_address[-1] == "]":
ip_address = ip_address[1:-1]
result[ip_address].append({"dst_port": port, "listen_port": listen_port, "ip_address": public_ip})
return result
def _filter_vdc_workloads(self):
zos = get_zos()
user_workloads = zos.workloads.list_workloads(self.identity_tid, next_action=NextAction.DEPLOY)
result = []
for workload in user_workloads:
if workload.info.workload_type not in VDC_WORKLOAD_TYPES:
continue
if not workload.info.description:
continue
try:
description = j.data.serializers.json.loads(workload.info.description)
except:
continue
if description.get("vdc_uuid") != self.solution_uuid:
continue
result.append(workload)
return result
def get_vdc_workloads(self, load_proxy=False):
kubernetes = []
s3 = S3()
etcd = []
vmachines = []
threebot = VDCThreebot()
proxies = []
for workload in self._filter_vdc_workloads():
if workload.info.workload_type == WorkloadType.Kubernetes:
kubernetes.append(KubernetesNode.from_workload(workload))
elif workload.info.workload_type == WorkloadType.Container:
if "minio" in workload.flist:
container = S3Container.from_workload(workload)
s3.minio = container
elif "js-sdk" in workload.flist:
container = VDCThreebot.from_workload(workload)
threebot = container
elif "etcd" in workload.flist:
node = ETCDNode.from_workload(workload)
etcd.append(node)
elif workload.info.workload_type == WorkloadType.Zdb:
zdb = S3ZDB.from_workload(workload)
if zdb:
s3.zdbs.append(zdb)
elif workload.info.workload_type == WorkloadType.Virtual_Machine:
vmachine = VMachine.from_workload(workload)
vmachines.append(vmachine)
elif workload.info.workload_type == WorkloadType.Subdomain:
s3_domain = self._check_s3_subdomains(workload)
if s3_domain:
s3.domain = workload.domain
s3.domain_wid = workload.id
elif workload.info.workload_type == WorkloadType.Reverse_proxy:
proxies.append(workload)
threebot.domain = self._get_threebot_subdomain(proxies, threebot)
if load_proxy:
self._build_zdb_proxies(s3)
return kubernetes, s3, vmachines, etcd, threebot
def _check_s3_subdomains(self, workload):
minio_wid = self.s3.minio.wid
if not minio_wid:
return
if not workload.info.description:
return
try:
desc = j.data.serializers.json.loads(workload.info.description)
except Exception as e:
j.logger.warning(f"Failed to load workload {workload.id} description due to error {e}")
return
exposed_wid = desc.get("exposed_wid")
if exposed_wid == minio_wid:
return True
def _get_threebot_subdomain(self, proxy_workloads, threebot):
threebot_wid = threebot.wid
if not threebot_wid:
return
non_matching_domains = []
for workload in proxy_workloads:
if not workload.info.description:
continue
try:
desc = j.data.serializers.json.loads(workload.info.description)
except Exception as e:
j.logger.warning(f"Failed to load workload {workload.id} description due to error {e}")
continue
exposed_wid = desc.get("exposed_wid")
if exposed_wid == threebot_wid:
return workload.domain
else:
non_matching_domains.append(workload.domain)
if not threebot.domain and non_matching_domains:
return non_matching_domains[-1]
def apply_grace_period_action(self):
self.load_info()
j.logger.info(f"VDC: {self.solution_uuid}, GRACE_PERIOD_ACTION: initialization")
for k8s in self.kubernetes:
ip_address = k8s.public_ip
if ip_address == "::/128":
continue
try:
ssh_client = self.get_ssh_client(self.instance_name, user="******", ip_address=str(ip_address))
rc, out, err = ssh_client.sshclient.run("sudo ip link set cni0 down", warn=True)
if rc:
j.logger.critical(
f"VDC: {self.solution_uuid}, GRACE_PERIOD_ACTION: failed to shutdown cni0 wid: {k8s.wid}, rc: {rc}, out: {out}, err: {err}"
)
if k8s.role == KubernetesRole.MASTER:
rc, out, err = ssh_client.sshclient.run(
"sudo iptables -A INPUT -p tcp --destination-port 6443 -j DROP", warn=True
)
if rc:
j.logger.critical(
f"VDC: {self.solution_uuid}, GRACE_PERIOD_ACTION: failed to block kubernetes API wid: {k8s.wid}, rc: {rc}, out: {out}, err: {err}"
)
j.clients.sshclient.delete(self.instance_name)
except Exception as e:
j.logger.critical(
f"VDC: {self.solution_uuid}, GRACE_PERIOD_ACTION: failed to connect to kubernetes controller due to error {str(e)}"
)
raise e
self.is_blocked = True
self.save()
j.logger.info(f"VDC: {self.solution_uuid}, GRACE_PERIOD_ACTION: applied successfully")
def revert_grace_period_action(self):
self.load_info()
j.logger.info(f"VDC: {self.solution_uuid}, REVERT_GRACE_PERIOD_ACTION: intializing")
self.is_blocked = False
for k8s in self.kubernetes:
ip_address = k8s.public_ip
if ip_address == "::/128":
continue
try:
ssh_client = self.get_ssh_client(self.instance_name, user="******", ip_address=str(ip_address))
rc, out, err = ssh_client.sshclient.run("sudo ip link set cni0 up", warn=True)
if rc:
j.logger.critical(
f"VDC: {self.solution_uuid}, GRACE_PERIOD_ACTION: failed to bring up cni0 wid: {k8s.wid}, rc: {rc}, out: {out}, err: {err}"
)
if k8s.role == KubernetesRole.MASTER:
rc, out, err = ssh_client.sshclient.run(
"sudo iptables -D INPUT -p tcp --destination-port 6443 -j DROP", warn=True
)
if rc:
j.logger.critical(
f"VDC: {self.solution_uuid}, GRACE_PERIOD_ACTION: failed to unblock kubernetes API wid: {k8s.wid}, rc: {rc}, out: {out}, err: {err}"
)
j.clients.sshclient.delete(self.instance_name)
except Exception as e:
j.logger.critical(
f"VDC: {self.solution_uuid}, GRACE_PERIOD_ACTION: failed to connect to kubernetes controller due to error {str(e)}"
)
self.is_blocked = True
continue
self.save()
j.logger.info(f"VDC: {self.solution_uuid}, REVERT_GRACE_PERIOD_ACTION: reverted successfully")
def show_vdc_payment(self, bot, expiry=5, wallet_name=None):
discount = FARM_DISCOUNT.get()
amount = VDC_SIZE.PRICES["plans"][self.flavor] * (1 - discount)
payment_id, _ = j.sals.billing.submit_payment(
amount=amount,
wallet_name=wallet_name or self.prepaid_wallet.instance_name,
refund_extra=False,
expiry=expiry,
description=j.data.serializers.json.dumps(
{"type": "VDC_INIT", "owner": self.owner_tname, "solution_uuid": self.solution_uuid}
),
)
if amount > 0:
notes = []
if discount:
notes = ["For testing purposes, we applied a discount of {:.0f}%".format(discount * 100)]
return j.sals.billing.wait_payment(payment_id, bot=bot, notes=notes), amount, payment_id
else:
return True, amount, payment_id
def show_external_node_payment(self, bot, farm_name, size, no_nodes=1, expiry=5, wallet_name=None, public_ip=False):
discount = FARM_DISCOUNT.get()
duration = self.calculate_expiration_value() - j.data.time.utcnow().timestamp
month = 60 * 60 * 24 * 30
if duration > month:
duration = month
zos = j.sals.zos.get()
farm_id = zos._explorer.farms.get(farm_name=farm_name).id
k8s = K8s()
if isinstance(size, str):
size = VDC_SIZE.K8SNodeFlavor[size.upper()].value
k8s.size = size
amount = j.tools.zos.consumption.cost(k8s, duration, farm_id) + TRANSACTION_FEES
if public_ip:
pub_ip = PublicIP()
amount += j.tools.zos.consumption.cost(pub_ip, duration, farm_id)
amount *= no_nodes
prepaid_balance = self._get_wallet_balance(self.prepaid_wallet)
if prepaid_balance >= amount:
if bot:
result = bot.single_choice(
f"Do you want to use your existing balance to pay {round(amount,4)} TFT? (This will impact the overall expiration of your plan)",
["Yes", "No"],
required=True,
)
if result == "Yes":
amount = 0
else:
amount = 0
elif not bot:
# Not enough funds in prepaid wallet and no bot passed to use to view QRcode
return False, amount, None
payment_id, _ = j.sals.billing.submit_payment(
amount=amount,
wallet_name=wallet_name or self.prepaid_wallet.instance_name,
refund_extra=False,
expiry=expiry,
description=j.data.serializers.json.dumps(
{"type": "VDC_K8S_EXTEND", "owner": self.owner_tname, "solution_uuid": self.solution_uuid}
),
)
if amount > 0:
notes = []
if discount:
notes = ["For testing purposes, we applied a discount of {:.0f}%".format(discount * 100)]
return j.sals.billing.wait_payment(payment_id, bot=bot, notes=notes), amount, payment_id
else:
return True, amount, payment_id
def show_external_zdb_payment(
self, bot, farm_name, size=ZDB_STARTING_SIZE, no_nodes=1, expiry=5, wallet_name=None, disk_type=DiskType.HDD
):
discount = FARM_DISCOUNT.get()
duration = self.calculate_expiration_value() - j.data.time.utcnow().timestamp
month = 60 * 60 * 24 * 30
if duration > month:
duration = month
zos = j.sals.zos.get()
farm_id = zos._explorer.farms.get(farm_name=farm_name).id
zdb = ZdbNamespace()
zdb.size = size
zdb.disk_type = disk_type
amount = j.tools.zos.consumption.cost(zdb, duration, farm_id) + TRANSACTION_FEES
amount *= no_nodes
prepaid_balance = self._get_wallet_balance(self.prepaid_wallet)
if prepaid_balance >= amount:
if bot:
result = bot.single_choice(
f"Do you want to use your existing balance to pay {round(amount,4)} TFT? (This will impact the overall expiration of your plan)",
["Yes", "No"],
required=True,
)
if result == "Yes":
amount = 0
else:
amount = 0
elif not bot:
# Not enough funds in prepaid wallet and no bot passed to use to view QRcode
return False, amount, None
payment_id, _ = j.sals.billing.submit_payment(
amount=amount,
wallet_name=wallet_name or self.prepaid_wallet.instance_name,
refund_extra=False,
expiry=expiry,
description=j.data.serializers.json.dumps(
{"type": "VDC_ZDB_EXTEND", "owner": self.owner_tname, "solution_uuid": self.solution_uuid}
),
)
if amount > 0:
notes = []
if discount:
notes = ["For testing purposes, we applied a discount of {:.0f}%".format(discount * 100)]
return j.sals.billing.wait_payment(payment_id, bot=bot, notes=notes), amount, payment_id
else:
return True, amount, payment_id
def show_external_vmachine_payment(self, bot, farm_name, size_number, expiry=5, wallet_name=None, public_ip=False):
discount = FARM_DISCOUNT.get()
duration = self.calculate_expiration_value() - j.data.time.utcnow().timestamp
month = 60 * 60 * 24 * 30
if duration > month:
duration = month
zos = j.sals.zos.get()
farm_id = zos._explorer.farms.get(farm_name=farm_name).id
vmachine = VirtualMachine()
vmachine.size = size_number
amount = j.tools.zos.consumption.cost(vmachine, duration, farm_id) + TRANSACTION_FEES
if public_ip:
pub_ip = PublicIP()
amount += j.tools.zos.consumption.cost(pub_ip, duration, farm_id)
prepaid_balance = self._get_wallet_balance(self.prepaid_wallet)
if prepaid_balance >= amount:
if bot:
result = bot.single_choice(
f"Do you want to use your existing balance to pay {round(amount,4)} TFT? (This will impact the overall expiration of your plan)",
["Yes", "No"],
required=True,
)
if result == "Yes":
amount = 0
else:
amount = 0
elif not bot:
# Not enough funds in prepaid wallet and no bot passed to use to view QRcode
return False, amount, None
payment_id, _ = j.sals.billing.submit_payment(
amount=amount,
wallet_name=wallet_name or self.prepaid_wallet.instance_name,
refund_extra=False,
expiry=expiry,
description=j.data.serializers.json.dumps(
{"type": "VM_CREATION", "owner": self.owner_tname, "solution_uuid": self.solution_uuid}
),
)
if amount > 0:
notes = []
if discount:
notes = ["For testing purposes, we applied a discount of {:.0f}%".format(discount * 100)]
return j.sals.billing.wait_payment(payment_id, bot=bot, notes=notes), amount, payment_id
else:
return True, amount, payment_id
def transfer_to_provisioning_wallet(self, amount, wallet_name=None):
if not amount:
return True
if wallet_name:
wallet = j.clients.stellar.get(wallet_name)
else:
wallet = self.prepaid_wallet
return self.pay_amount(self.provision_wallet.address, amount, wallet)
def pay_initialization_fee(self, transaction_hashes, initial_wallet_name, wallet_name=None):
initial_wallet = j.clients.stellar.get(initial_wallet_name)
if wallet_name:
wallet = j.clients.stellar.get(wallet_name)
else:
wallet = self.provision_wallet
# get total amount
amount = self._calculate_initialization_fee(transaction_hashes, initial_wallet)
if not amount:
return True
return self.pay_amount(initial_wallet.address, amount, wallet)
def _calculate_initialization_fee(self, transaction_hashes, initial_wallet):
amount = 0
for t_hash in transaction_hashes:
effects = initial_wallet.get_transaction_effects(t_hash)
for effect in effects:
amount += (
abs(float(effect.amount)) + TRANSACTION_FEES
) # transaction fees to not drain the initialization wallet
amount = round(amount, 6)
return amount
def _get_wallet_balance(self, wallet):
balances = wallet.get_balance().balances
for b in balances:
if b.asset_code != "TFT":
continue
return float(b.balance)
return 0
def pay_amount(self, address, amount, wallet, memo_text=""):
j.logger.info(f"transfering amount: {amount} from wallet: {wallet.instance_name} to address: {address}")
deadline = j.data.time.now().timestamp + 5 * 60
a = wallet._get_asset()
has_funds = None
while j.data.time.now().timestamp < deadline:
if not has_funds:
try:
balances = wallet.get_balance().balances
for b in balances:
if b.asset_code != "TFT":
continue
if amount <= float(b.balance) + TRANSACTION_FEES:
has_funds = True
break
else:
has_funds = False
raise j.exceptions.Validation(
f"Not enough funds in wallet {wallet.instance_name} to pay amount: {amount}. current balance: {b.balance}"
)
except Exception as e:
if has_funds is False:
j.logger.error(f"Not enough funds in wallet {wallet.instance_name} to pay amount: {amount}")
raise e
j.logger.warning(f"Failed to get wallet {wallet.instance_name} balance due to error: {str(e)}")
continue
try:
return wallet.transfer(
address, amount=round(amount, 6), asset=f"{a.code}:{a.issuer}", memo_text=memo_text
)
except Exception as e:
j.logger.warning(f"failed to submit payment to stellar due to error {str(e)}")
j.logger.critical(
f"Failed to submit payment to stellar in time to: {address} amount: {amount} for wallet: {wallet.instance_name}"
)
raise j.exceptions.Runtime(
f"Failed to submit payment to stellar in time to: {address} amount: {amount} for wallet: {wallet.instance_name}"
)
def get_pools_expiration(self):
active_pools = self.active_pools
if not active_pools:
return 0
if len(active_pools) < 2:
return active_pools[0].empty_at
return min(*[p.empty_at for p in active_pools])
def get_total_funds(self):
total_tfts = 0
for wallet in [self.prepaid_wallet, self.provision_wallet]:
for balance in wallet.get_balance().balances:
if balance.asset_code == "TFT":
total_tfts += float(balance.balance)
break
return total_tfts
def get_current_spec(self, load_info=True):
"""
return
dict:
{
"plan": self.flavor,
"nodes": [additional_nodes_flavors],
"services": {
"services.IP": count
}
"no_nodes": no_nodes # that were not deployed during initial vdc deployment
}
"""
if load_info:
self.load_info()
result = {"plan": self.flavor, "nodes": [], "services": {VDC_SIZE.Services.IP: 0}}
no_nodes = VDC_SIZE.VDC_FLAVORS[self.flavor]["k8s"]["no_nodes"]
node_flavor = VDC_SIZE.VDC_FLAVORS[self.flavor]["k8s"]["size"]
for k8s in self.kubernetes:
if k8s.role == KubernetesRole.MASTER:
continue
metadata = self._decrypt_metadata(k8s.wid)
if k8s.size == node_flavor and no_nodes > 0 and not metadata.get("external"):
no_nodes -= 1
continue
result["nodes"].append(k8s.size)
if k8s.public_ip != "::/128":
result["services"][VDC_SIZE.Services.IP] += 1
result["no_nodes"] = no_nodes
return result
def _decrypt_metadata(self, wid, workload=None):
identity = self._get_identity()
workload = workload or j.sals.zos.get(identity.instance_name).workloads.get(wid)
metadata = j.sals.reservation_chatflow.deployer.decrypt_metadata(workload.info.metadata, identity.instance_name)
return j.data.serializers.json.loads(metadata)
def calculate_spec_price(self, load_info=True):
discount = FARM_DISCOUNT.get()
current_spec = self.get_current_spec(load_info)
total_price = (
VDC_SIZE.PRICES["plans"][self.flavor]
+ current_spec["services"][VDC_SIZE.Services.IP] * VDC_SIZE.PRICES["services"][VDC_SIZE.Services.IP]
)
for size in current_spec["nodes"]:
total_price += VDC_SIZE.PRICES["nodes"][size]
return total_price * (1 - discount)
def calculate_funded_period(self, load_info=True):
"""
return how many days can the vdc be extended with prepaid + provisioning wallet
"""
if load_info:
self.load_info()
prepaid_wallet_balance = self._get_wallet_balance(self.prepaid_wallet)
provision_wallet_balance = self._get_wallet_balance(self.provision_wallet)
days_prepaid_can_fund = (prepaid_wallet_balance / self.calculate_spec_price(load_info)) * 30
days_provisioning_can_fund = provision_wallet_balance / (self.calculate_active_units_price() * 60 * 60 * 24)
return days_prepaid_can_fund + days_provisioning_can_fund
def calculate_active_units_price(self):
cus = sus = ipv4us = 0
for pool in self.active_pools:
cus += pool.active_cu
sus += pool.active_su
ipv4us += pool.active_ipv4
return self._get_identity().explorer.prices.calculate(cus, sus, ipv4us) # TFTs per second
def calculate_expiration_value(self, load_info=True):
funded_period = self.calculate_funded_period(load_info)
pools_expiration = self.get_pools_expiration()
return pools_expiration + funded_period * 24 * 60 * 60
def fund_difference(self, funding_wallet_name, destination_wallet_name=None):
wallet = j.clients.stellar.find(funding_wallet_name)
destination_wallet_name = destination_wallet_name or self.provision_wallet.instance_name
dst_wallet = j.clients.stellar.find(destination_wallet_name)
current_balance = self._get_wallet_balance(dst_wallet)
j.logger.info(f"current balance in {destination_wallet_name} is {current_balance}")
vdc_cost = float(j.tools.zos.consumption.calculate_vdc_price(self.flavor.value)) / 2
j.logger.info(f"vdc_cost: {vdc_cost}")
if vdc_cost > current_balance:
diff = float(vdc_cost) - float(current_balance)
j.logger.info(f"funding diff: {diff} for vdc {self.vdc_name} from wallet: {funding_wallet_name}")
self.pay_amount(dst_wallet.address, diff, wallet)
def get_ssh_client(self, name, ip_address, user, private_key_path=None):
private_key_path = (
private_key_path or f"{j.core.dirs.CFGDIR}/vdc/keys/{self.owner_tname}/{self.vdc_name}/id_rsa"
)
if not j.sals.fs.exists(private_key_path):
private_key_path = "/root/.ssh/id_rsa"
if not j.sals.fs.exists(private_key_path):
raise j.exceptions.Input(f"couldn't find key at default locations")
j.logger.info(f"getting ssh_client to: {user}@{ip_address} using key: {private_key_path}")
client_name = SSH_KEY_PREFIX + name
j.clients.sshkey.delete(client_name)
ssh_key = j.clients.sshkey.get(client_name)
ssh_key.private_key_path = private_key_path
ssh_key.load_from_file_system()
j.clients.sshclient.delete(client_name)
ssh_client = j.clients.sshclient.get(client_name, user=user, host=ip_address, sshkey=client_name)
return ssh_client
def find_worker_farm(self, flavor, farm_name=None, public_ip=False):
if farm_name:
return farm_name, self._check_added_worker_capacity(flavor, farm_name, public_ip)
farms = j.config.get("NETWORK_FARMS", []) if public_ip else j.config.get("COMPUTE_FARMS", [])
for farm in farms:
if self._check_added_worker_capacity(flavor, farm, public_ip):
return farm, True
else:
self.load_info()
pool_id = [n for n in self.kubernetes if n.role == KubernetesRole.MASTER][-1].pool_id
farm_name = j.sals.marketplace.deployer.get_pool_farm_name(pool_id)
return farm_name, self._check_added_worker_capacity(flavor, farm_name, public_ip)
def have_capacity(self, query, vdc, farm_name=None, public_ip=False):
if public_ip:
zos = j.sals.zos.get()
farm = zos._explorer.farms.get(farm_name=farm_name)
available_ips = False
for address in farm.ipaddresses:
if not address.reservation_id:
available_ips = True
break
if not available_ips:
return False
old_node_ids = []
vdc.load_info()
for k8s_node in vdc.kubernetes:
old_node_ids.append(k8s_node.node_id)
for vmachine in vdc.vmachines:
old_node_ids.append(vmachine.node_id)
cc = CapacityChecker(farm_name)
cc.exclude_nodes(*old_node_ids)
if not cc.add_query(**query):
return False
return True
def find_vmachine_farm(self, query, farm_name=None, public_ip=False):
if farm_name:
return farm_name, self.have_capacity(query=query, vdc=self, farm_name=farm_name, public_ip=public_ip)
farms = j.config.get("NETWORK_FARMS", []) if public_ip else j.config.get("COMPUTE_FARMS", [])
for farm in farms:
if self.have_capacity(query=query, vdc=self, farm_name=farm, public_ip=public_ip):
return farm, True
else:
self.load_info()
pool_id = [n for n in self.kubernetes if n.role == KubernetesRole.MASTER][-1].pool_id
farm_name = j.sals.marketplace.deployer.get_pool_farm_name(pool_id)
return farm_name, self.have_capacity(query=query, vdc=self, farm_name=farm_name, public_ip=public_ip)
def _check_added_worker_capacity(self, flavor, farm_name, public_ip=False):
if public_ip:
zos = j.sals.zos.get()
farm = zos._explorer.farms.get(farm_name=farm_name)
available_ips = False
for address in farm.ipaddresses:
if not address.reservation_id:
available_ips = True
break
if not available_ips:
return False
old_node_ids = []
self.load_info()
for k8s_node in self.kubernetes:
old_node_ids.append(k8s_node.node_id)
cc = CapacityChecker(farm_name)
cc.exclude_nodes(*old_node_ids)
if isinstance(flavor, str):
flavor = VDC_SIZE.K8SNodeFlavor[flavor.upper()]
if not cc.add_query(**VDC_SIZE.K8S_SIZES[flavor]):
return False
return True
class ContainerCapacity(Base):
cpu = fields.Integer()
memory = fields.Integer()
disk_size = fields.Integer()
disk_type = fields.Enum(DiskType)
class Stellar(Client):
network = fields.Enum(Network)
address = fields.String()
def secret_updated(self, value):
self.address = stellar_sdk.Keypair.from_secret(value).public_key
secret = fields.String(on_update=secret_updated)
def _get_horizon_server(self):
server_url = _HORIZON_NETWORKS[self.network.value]
server = Server(horizon_url=server_url)
return server
def _get_free_balances(self, address=None):
address = address or self.address
balances = AccountBalances(address)
response = self._get_horizon_server().accounts().account_id(address).call()
for response_balance in response["balances"]:
balances.add_balance(Balance.from_horizon_response(response_balance))
return balances
def load_account(self):
horizonServer = self._get_horizon_server()
saccount = horizonServer.load_account(self.address)
account = Account(saccount.account_id, saccount.sequence, self)
return account
def _get_url(self, endpoint):
url = _THREEFOLDFOUNDATION_TFTSTELLAR_SERVICES[self.network.value]
if not j.sals.nettools.wait_connection_test(url, 443, 5):
raise j.exceptions.Timeout(f"Can not connect to server {url}, connection timeout")
endpoint = _THREEFOLDFOUNDATION_TFTSTELLAR_ENDPOINT[endpoint]
return f"https://{url}{endpoint}"
def _fund_transaction(self, transaction):
data = {"transaction": transaction}
resp = j.tools.http.post(self._get_url("FUND"), json={"args": data})
resp.raise_for_status()
return resp.json()
def _create_unlockhash_transaction(self, unlock_hash, transaction_xdr):
data = {"unlockhash": unlock_hash, "transaction_xdr": transaction_xdr}
resp = j.tools.http.post(self._get_url("CREATE_UNLOCK"), json={"args": data})
resp.raise_for_status()
return resp.json()
def _get_unlockhash_transaction(self, unlockhash):
data = {"unlockhash": unlockhash}
resp = j.tools.http.post(self._get_url("GET_UNLOCK"), json={"args": data})
resp.raise_for_status()
return resp.json()
def _create_activation_code(self):
data = {"address": self.address}
resp = j.tools.http.post(self._get_url("CREATE_ACTIVATION_CODE"), json={"args": data})
resp.raise_for_status()
return resp.json()
def _activation_account(self, activation_code):
data = {"activation_code": activation_code}
resp = j.tools.http.post(self._get_url("ACTIVATE_ACCOUNT"), json={"args": data})
resp.raise_for_status()
return resp.json()
def set_unlock_transaction(self, unlock_transaction):
"""
Adds a xdr encoded unlocktransaction
:param unlock_transaction: xdr encoded unlocktransactionaddress of the destination.
:type destination_address: str
"""
txe = stellar_sdk.TransactionEnvelope.from_xdr(unlock_transaction, _NETWORK_PASSPHRASES[self.network.value])
tx_hash = txe.hash()
unlock_hash = stellar_sdk.strkey.StrKey.encode_pre_auth_tx(tx_hash)
self._create_unlockhash_transaction(unlock_hash=unlock_hash, transaction_xdr=txe.to_xdr())
def get_balance(self, address=None):
"""Gets balance for a stellar address
"""
if address is None:
address = self.address
all_balances = self._get_free_balances(address)
for account in self._find_escrow_accounts(address):
all_balances.add_escrow_account(account)
return all_balances
def _find_escrow_accounts(self, address=None):
if address is None:
address = self.address
escrow_accounts = []
accounts_endpoint = self._get_horizon_server().accounts()
accounts_endpoint.signer(address)
old_cursor = "old"
new_cursor = ""
while new_cursor != old_cursor:
old_cursor = new_cursor
accounts_endpoint.cursor(new_cursor)
response = accounts_endpoint.call()
next_link = response["_links"]["next"]["href"]
next_link_query = parse.urlsplit(next_link).query
new_cursor = parse.parse_qs(next_link_query)["cursor"][0]
accounts = response["_embedded"]["records"]
for account in accounts:
account_id = account["account_id"]
if account_id == address:
continue # Do not take the receiver's account
all_signers = account["signers"]
preauth_signers = [signer["key"] for signer in all_signers if signer["type"] == "preauth_tx"]
# TODO check the tresholds and signers
# TODO if we can merge, the amount is unlocked ( if len(preauth_signers))==0
balances = []
for response_balance in account["balances"]:
balances.append(Balance.from_horizon_response(response_balance))
escrow_account = EscrowAccount(
account_id,
preauth_signers,
balances,
_NETWORK_PASSPHRASES[self.network.value],
self._get_unlockhash_transaction,
)
escrow_accounts.append(escrow_account)
return escrow_accounts
def claim_locked_funds(self):
balances = self.get_balance()
for locked_account in balances.escrow_accounts:
if locked_account.can_be_unlocked():
self._unlock_account(locked_account)
def _unlock_account(self, escrow_account):
submitted_unlock_transactions = 0
for unlockhash in escrow_account.unlockhashes:
unlockhash_transation = self._get_unlockhash_transaction(unlockhash=unlockhash)
if unlockhash_transation is None:
return
j.logger.info(unlockhash_transation["transaction_xdr"])
self._get_horizon_server().submit_transaction(unlockhash_transation["transaction_xdr"])
submitted_unlock_transactions += 1
if submitted_unlock_transactions == len(escrow_account.unlockhashes):
self._merge_account(escrow_account.address)
def _merge_account(self, address):
server = self._get_horizon_server()
account = server.load_account(address)
# Increment the sequence number in case the unlock transaction was not processed before the load_account call
# account.increment_sequence_number()
balances = self._get_free_balances(address)
base_fee = server.fetch_base_fee()
transaction_builder = stellar_sdk.TransactionBuilder(
source_account=account, network_passphrase=_NETWORK_PASSPHRASES[self.network.value], base_fee=base_fee
)
for balance in balances.balances:
if balance.is_native():
continue
# Step 1: Transfer custom assets
transaction_builder.append_payment_op(
destination=self.address,
amount=balance.balance,
asset_code=balance.asset_code,
asset_issuer=balance.asset_issuer,
)
# Step 2: Delete trustlines
transaction_builder.append_change_trust_op(
asset_issuer=balance.asset_issuer, asset_code=balance.asset_code, limit="0"
)
# Step 3: Merge account
transaction_builder.append_account_merge_op(self.address)
transaction_builder.set_timeout(30)
transaction = transaction_builder.build()
signer_kp = stellar_sdk.Keypair.from_secret(self.secret)
transaction.sign(signer_kp)
server.submit_transaction(transaction)
def activate_through_friendbot(self):
"""Activates and funds a testnet account using riendbot
"""
if self.network.value != "TEST":
raise Exception("Account activation through friendbot is only available on testnet")
resp = j.tools.http.get("https://friendbot.stellar.org/", params={"addr": self.address})
resp.raise_for_status()
j.logger.info(f"account with address {self.address} activated and funded through friendbot")
def activate_through_threefold_service(self):
"""
Activate your weallet through threefold services
"""
activationdata = self._create_activation_code()
self._activation_account(activationdata["activation_code"])
def activate_account(self, destination_address, starting_balance="12.50"):
"""Activates another account
Args:
destination_address (str): address of the destination
starting_balance (str, optional): the balance that the destination address will start with. Must be a positive integer expressed as a string. Defaults to "12.50".
"""
server = self._get_horizon_server()
source_keypair = stellar_sdk.Keypair.from_secret(self.secret)
source_account = self.load_account()
base_fee = server.fetch_base_fee()
transaction = (
stellar_sdk.TransactionBuilder(
source_account=source_account,
network_passphrase=_NETWORK_PASSPHRASES[self.network.value],
base_fee=base_fee,
)
.append_create_account_op(destination=destination_address, starting_balance=starting_balance)
.build()
)
transaction.sign(source_keypair)
try:
response = server.submit_transaction(transaction)
j.logger.info("Transaction hash: {}".format(response["hash"]))
except stellar_sdk.exceptions.BadRequestError as e:
j.logger.debug(e)
def add_trustline(self, asset_code, issuer, secret=None):
"""Create a trustline to an asset
Args:
asset_code (str): code of the asset. For example: 'BTC', 'TFT', ...
issuer (str): address of the asset issuer
secret (str, optional): Secret to use will use instance property if empty. Defaults to None.
"""
self._change_trustline(asset_code, issuer, secret=secret)
def add_known_trustline(self, asset_code):
"""Will add a trustline known by threefold for chosen asset_code
Args:
asset_code (str): code of the asset. For example: 'BTC', 'TFT', ...
"""
issuer = _NETWORK_KNOWN_TRUSTS.get(self.network.value, {}).get(asset_code)
if not issuer:
raise j.exceptions.NotFound(f"We do not provide a known issuers for {asset_code} on network {self.network}")
self._change_trustline(asset_code, issuer)
def delete_trustline(self, asset_code, issuer, secret=None):
"""Deletes a trustline
Args:
asset_code (str): code of the asset. For example: 'BTC', 'XRP', ...
issuer (str): address of the asset issuer
secret (str, optional): Secret to use will use instance property if empty. Defaults to None.
"""
self._change_trustline(asset_code, issuer, limit="0", secret=secret)
def _change_trustline(self, asset_code, issuer, limit=None, secret=None):
"""Create a trustline between you and the issuer of an asset
Args:
asset_code (str): code which form the asset. For example: 'BTC', 'TFT', ...
issuer (str): address of the asset issuer
limit ([type], optional): The limit for the asset, defaults to max int64(922337203685.4775807). If the limit is set to “0” it deletes the trustline. Defaults to None.
secret (str, optional): Secret to use will use instance property if empty. Defaults to None.
"""
# if no secret is provided we assume we change trustlines for this account
secret = secret or self.secret
server = self._get_horizon_server()
source_keypair = stellar_sdk.Keypair.from_secret(secret)
source_public_key = source_keypair.public_key
source_account = server.load_account(source_public_key)
base_fee = server.fetch_base_fee()
transaction = (
stellar_sdk.TransactionBuilder(
source_account=source_account,
network_passphrase=_NETWORK_PASSPHRASES[self.network.value],
base_fee=base_fee,
)
.append_change_trust_op(asset_issuer=issuer, asset_code=asset_code, limit=limit)
.set_timeout(30)
.build()
)
transaction.sign(source_keypair)
try:
response = server.submit_transaction(transaction)
j.logger.info("Transaction hash: {}".format(response["hash"]))
except stellar_sdk.exceptions.BadRequestError as e:
j.logger.debug(e)
raise e
def transfer(
self,
destination_address,
amount,
asset="XLM",
locked_until=None,
memo_text=None,
memo_hash=None,
fund_transaction=True,
from_address=None,
timeout=30,
sequence_number: int = None,
sign: bool = True,
):
"""Transfer assets to another address
Args:
destination_address (str): address of the destination
amount (str): can be a floating point number with 7 numbers after the decimal point expressed as a string
asset (str, optional): asset to transfer. Defaults to "XLM". if you wish to specify an asset it should be in format 'assetcode:issuer'. Where issuer is the address of the
issuer of the asset.
locked_until (float, optional): epoch timestamp indicating until when the tokens should be locked. Defaults to None.
memo_text (Union[str, bytes], optional): memo text to add to the transaction, a string encoded using either ASCII or UTF-8, up to 28-bytes long. Defaults to None.
memo_hash (Union[str, bytes], optional): memo hash to add to the transaction, A 32 byte hash. Defaults to None.
fund_transaction (bool, optional): use the threefoldfoundation transaction funding service. Defautls to True.
from_address (str, optional): Use a different address to send the tokens from, useful in multisig use cases. Defaults to None.
timeout (int,optional: Seconds from now on until when the transaction to be submitted to the stellar network
sequence_number (int,optional): specify a specific sequence number ( will still be increased by one) instead of loading it from the account
sign (bool,optional) : Do not sign and submit the transaction
Raises:
Exception: If asset not in correct format
stellar_sdk.exceptions.BadRequestError: not enough funds for opertaion
stellar_sdk.exceptions.BadRequestError: bad transfer authentication
Returns:
[type]: [description]
"""
issuer = None
j.logger.info(f"Sending {amount} {asset} to {destination_address}")
if asset != "XLM":
assetStr = asset.split(":")
if len(assetStr) != 2:
raise Exception("Wrong asset format should be in format 'assetcode:issuer'")
asset = assetStr[0]
issuer = assetStr[1]
if locked_until is not None:
return self._transfer_locked_tokens(
destination_address,
amount,
asset,
issuer,
locked_until,
memo_text=memo_text,
memo_hash=memo_hash,
fund_transaction=fund_transaction,
from_address=from_address,
)
horizon_server = self._get_horizon_server()
base_fee = horizon_server.fetch_base_fee()
if from_address:
source_account = horizon_server.load_account(from_address)
else:
source_account = self.load_account()
if sequence_number:
source_account.sequence = sequence_number
transaction_builder = stellar_sdk.TransactionBuilder(
source_account=source_account,
network_passphrase=_NETWORK_PASSPHRASES[self.network.value],
base_fee=base_fee,
)
transaction_builder.append_payment_op(
destination=destination_address,
amount=str(amount),
asset_code=asset,
asset_issuer=issuer,
source=source_account.account_id,
)
transaction_builder.set_timeout(timeout)
if memo_text is not None:
transaction_builder.add_text_memo(memo_text)
if memo_hash is not None:
transaction_builder.add_hash_memo(memo_hash)
transaction = transaction_builder.build()
transaction = transaction.to_xdr()
if asset == "TFT" or asset == "FreeTFT":
if fund_transaction:
transaction = self._fund_transaction(transaction=transaction)
transaction = transaction["transaction_xdr"]
if not sign:
return transaction
transaction = stellar_sdk.TransactionEnvelope.from_xdr(transaction, _NETWORK_PASSPHRASES[self.network.value])
my_keypair = stellar_sdk.Keypair.from_secret(self.secret)
transaction.sign(my_keypair)
response = horizon_server.submit_transaction(transaction)
tx_hash = response["hash"]
j.logger.info(f"Transaction hash: {tx_hash}")
return tx_hash
def list_payments(self, address: str = None, asset: str = None, cursor: str = None):
"""Get the transactions for an adddress
:param address: address of the effects.In None, the address of this wallet is taken
:param asset: stellar asset in the code:issuer form( except for XLM, which does not need an issuer)
:param cursor:pass a cursor to continue after the last call or an empty str to start receivibg a cursor
if a cursor is passed, a tuple of the payments and the cursor is returned
"""
if address is None:
address = self.address
tx_endpoint = self._get_horizon_server().payments()
tx_endpoint.for_account(address)
tx_endpoint.limit(50)
payments = []
old_cursor = "old"
new_cursor = ""
if cursor is not None:
new_cursor = cursor
while old_cursor != new_cursor:
old_cursor = new_cursor
tx_endpoint.cursor(new_cursor)
response = tx_endpoint.call()
next_link = response["_links"]["next"]["href"]
next_link_query = parse.urlsplit(next_link).query
new_cursor = parse.parse_qs(next_link_query)["cursor"][0]
response_payments = response["_embedded"]["records"]
for response_payment in response_payments:
ps = PaymentSummary.from_horizon_response(response_payment, address)
if asset:
split_asset = asset.split(":")
assetcode = split_asset[0]
assetissuer = None
if len(split_asset) > 1:
assetissuer = split_asset[1]
if ps.balance and ps.balance.asset_code == assetcode:
if assetissuer and assetissuer == ps.balance.asset_issuer:
payments.append(ps)
else:
payments.append(ps)
if cursor is not None:
return {"payments": payments, "cursor": new_cursor}
return payments
def list_transactions(self, address: str = None, cursor: str = None):
"""Get the transactions for an adddres
:param address (str, optional): address of the effects.If None, the address of this wallet is taken. Defaults to None.
:param cursor:pass a cursor to continue after the last call or an empty str to start receivibg a cursor
if a cursor is passed, a tuple of the payments and the cursor is returned
Returns:
list: list of TransactionSummary objects
dictionary: {"transactions":list of TransactionSummary objects, "cursor":cursor}
"""
address = address or self.address
tx_endpoint = self._get_horizon_server().transactions()
tx_endpoint.for_account(address)
tx_endpoint.include_failed(True)
transactions = []
old_cursor = "old"
new_cursor = ""
if cursor is not None:
new_cursor = cursor
while old_cursor != new_cursor:
old_cursor = new_cursor
tx_endpoint.cursor(new_cursor)
response = tx_endpoint.call()
next_link = response["_links"]["next"]["href"]
next_link_query = parse.urlsplit(next_link).query
new_cursor = parse.parse_qs(next_link_query)["cursor"][0]
response_transactions = response["_embedded"]["records"]
for response_transaction in response_transactions:
if response_transaction["successful"]:
transactions.append(TransactionSummary.from_horizon_response(response_transaction))
if cursor is not None:
return {"transactions": transactions, "cursor": new_cursor}
return transactions
def get_transaction_effects(self, transaction_hash, address=None):
"""Get the effects on an adddressfor a specific transaction
Args:
transaction_hash (str): hash of the transaction
address (str, optional): address of the effects.If None, the address of this wallet is taken. Defaults to None.
Returns:
list: list of Effect objects
"""
address = address or self.address
effects = []
endpoint = self._get_horizon_server().effects()
endpoint.for_transaction(transaction_hash)
old_cursor = "old"
new_cursor = ""
while old_cursor != new_cursor:
old_cursor = new_cursor
endpoint.cursor(new_cursor)
response = endpoint.call()
next_link = response["_links"]["next"]["href"]
next_link_query = parse.urlsplit(next_link).query
new_cursor = parse.parse_qs(next_link_query)["cursor"][0]
response_effects = response["_embedded"]["records"]
for response_effect in response_effects:
if "account" in response_effect and response_effect["account"] == address:
effects.append(Effect.from_horizon_response(response_effect))
return effects
def _transfer_locked_tokens(
self,
destination_address,
amount,
asset_code,
asset_issuer,
unlock_time,
memo_text=None,
memo_hash=None,
fund_transaction=True,
from_address=None,
):
"""Transfer locked assets to another address
Args:
destination_address (str): address of the destination
amount (str): amount, can be a floating point number with 7 numbers after the decimal point expressed as a string
asset_code (str): asset to transfer
asset_issuer (str): if the asset_code is different from 'XlM', the issuer address
unlock_time (float): an epoch timestamp indicating when the funds should be unlocked
memo_text (Union[str, bytes], optional): memo text to add to the transaction, a string encoded using either ASCII or UTF-8, up to 28-bytes long
memo_hash (Union[str, bytes], optional): memo hash to add to the transaction, A 32 byte hash
fund_transaction (bool, optional): use the threefoldfoundation transaction funding service.Defaults to True.
from_address (str, optional): Use a different address to send the tokens from, useful in multisig use cases. Defaults to None.
Returns:
[type]: [description]
"""
unlock_time = math.ceil(unlock_time)
self._log_info("Creating escrow account")
escrow_kp = stellar_sdk.Keypair.random()
# minimum account balance as described at https://www.stellar.org/developers/guides/concepts/fees.html#minimum-account-balance
horizon_server = self._get_horizon_server()
base_fee = horizon_server.fetch_base_fee()
base_reserve = 0.5
minimum_account_balance = (2 + 1 + 3) * base_reserve # 1 trustline and 3 signers
required_XLM = minimum_account_balance + base_fee * 0.0000001 * 3
self._log_info("Activating escrow account")
self.activate_account(escrow_kp.public_key, str(math.ceil(required_XLM)))
if asset_code != "XLM":
self._log_info("Adding trustline to escrow account")
self.add_trustline(asset_code, asset_issuer, escrow_kp.secret)
preauth_tx = self._create_unlock_transaction(escrow_kp, unlock_time)
preauth_tx_hash = preauth_tx.hash()
# save the preauth transaction in our unlock service
unlock_hash = stellar_sdk.strkey.StrKey.encode_pre_auth_tx(preauth_tx_hash)
self._create_unlockhash_transaction(unlock_hash=unlock_hash, transaction_xdr=preauth_tx.to_xdr())
self._set_escrow_account_signers(escrow_kp.public_key, destination_address, preauth_tx_hash, escrow_kp)
self._log_info("Unlock Transaction:")
self._log_info(preauth_tx.to_xdr())
self.transfer(
escrow_kp.public_key,
amount,
asset_code + ":" + asset_issuer,
memo_text=memo_text,
memo_hash=memo_hash,
fund_transaction=fund_transaction,
from_address=from_address,
)
return preauth_tx.to_xdr()
def _create_unlock_transaction(self, escrow_kp, unlock_time):
server = self._get_horizon_server()
escrow_account = server.load_account(escrow_kp.public_key)
escrow_account.increment_sequence_number()
tx = (
stellar_sdk.TransactionBuilder(escrow_account)
.append_set_options_op(master_weight=0, low_threshold=1, med_threshold=1, high_threshold=1)
.add_time_bounds(unlock_time, 0)
.build()
)
tx.sign(escrow_kp)
return tx
def _set_account_signers(self, address, public_key_signer, preauth_tx_hash, signer_kp):
server = self._get_horizon_server()
if address == self.address:
account = self.load_account()
else:
account = server.load_account(address)
tx = (
stellar_sdk.TransactionBuilder(account)
.append_pre_auth_tx_signer(preauth_tx_hash, 1)
.append_ed25519_public_key_signer(public_key_signer, 1)
.append_set_options_op(master_weight=1, low_threshold=2, med_threshold=2, high_threshold=2)
.build()
)
tx.sign(signer_kp)
response = server.submit_transaction(tx)
j.logger.info(response)
j.logger.info(f"Set the signers of {address} to {public_key_signer} and {preauth_tx_hash}")
def get_signing_requirements(self, address: str = None):
address = address or self.address
response = self._get_horizon_server().accounts().account_id(address).call()
signing_requirements = {}
signing_requirements["thresholds"] = response["thresholds"]
signing_requirements["signers"] = response["signers"]
return signing_requirements
def modify_signing_requirements(
self, public_keys_signers, signature_count, low_treshold=0, high_treshold=2, master_weight=2
):
"""modify_signing_requirements sets to amount of signatures required for the creation of multisig account. It also adds
the public keys of the signer to this account
Args:
public_keys_signers (list): list of public keys of signers
signature_count (int): amount of signatures requires to transfer funds
low_treshold (int, optional): amount of signatures required for low security operations (transaction processing, allow trust, bump sequence). Defaults to 1.
high_treshold (int, optional): amount of signatures required for high security operations (set options, account merge). Defaults to 2.
master_weight (int, optional): A number from 0-255 (inclusive) representing the weight of the master key. If the weight of the master key is updated to 0, it is effectively disabled. Defaults to 2.
"""
server = self._get_horizon_server()
account = self.load_account()
source_keypair = stellar_sdk.Keypair.from_secret(self.secret)
transaction_builder = stellar_sdk.TransactionBuilder(account)
# set the signing options
transaction_builder.append_set_options_op(
low_threshold=low_treshold,
med_threshold=signature_count,
high_threshold=high_treshold,
master_weight=master_weight,
)
# For every public key given, add it as a signer to this account
for public_key_signer in public_keys_signers:
transaction_builder.append_ed25519_public_key_signer(public_key_signer, 1)
transaction_builder.set_timeout(30)
tx = transaction_builder.build()
tx.sign(source_keypair)
try:
response = server.submit_transaction(tx)
j.logger.info(response)
j.logger.info(f"Set the signers of {self.address} to require {signature_count} signers")
except stellar_sdk.exceptions.BadRequestError:
j.logger.info("Transaction need additional signatures in order to send")
return tx.to_xdr()
def sign(self, tx_xdr: str, submit: bool = True):
""" sign signs a transaction xdr and optionally submits it to the network
Args:
tx_xdr (str): transaction to sign in xdr format
submit (bool,optional): submit the transaction tro the Stellar network
"""
source_keypair = stellar_sdk.Keypair.from_secret(self.secret)
tx = stellar_sdk.TransactionEnvelope.from_xdr(tx_xdr, _NETWORK_PASSPHRASES[self.network.value])
tx.sign(source_keypair)
if submit:
horizon_server = self._get_horizon_server()
horizon_server.submit_transaction(tx)
else:
return tx.to_xdr()
def sign_multisig_transaction(self, tx_xdr):
"""sign_multisig_transaction signs a transaction xdr and tries to submit it to the network
Deprecated, use sign instead
Args:
tx_xdr (str): transaction to sign in xdr format
"""
try:
self.sign(tx_xdr)
j.logger.info("Multisig tx signed and sent")
except UnAuthorized as e:
j.logger.info("Transaction needs additional signatures in order to send")
return e.transaction_xdr
def remove_signer(self, public_key_signer):
"""remove_signer removes a public key as a signer from the source account
Args:
public_key_signer (str): public key of an account
"""
server = self._get_horizon_server()
account = self.load_account()
tx = stellar_sdk.TransactionBuilder(account).append_ed25519_public_key_signer(public_key_signer, 0).build()
source_keypair = stellar_sdk.Keypair.from_secret(self.secret)
tx.sign(source_keypair)
try:
response = server.submit_transaction(tx)
j.logger.info(response)
j.logger.info("Multisig tx signed and sent")
except stellar_sdk.exceptions.BadRequestError:
j.logger.info("Transaction need additional signatures in order to send")
return tx.to_xdr()
def get_sender_wallet_address(self, transaction_hash):
"""Get the sender's wallet address from a transaction hash
Args:
transaction_hash (String): Transaction hash
Returns:
String : Wallet Hash
"""
server = self._get_horizon_server()
endpoint = server.operations().for_transaction(transaction_hash)
response = endpoint.call()
# not possible for a transaction to have more than a source, so will take first one
wallet_address = response["_embedded"]["records"][0]["source_account"]
return wallet_address
def check_is_payment_transaction(self, transaction_hash):
"""Some transactions doesn't have an amount like activating the wallet
This helper method to help in iterating in transactions
Args:
transaction_hash (String): Transaction hash
Returns:
Bool: True if transaction has amount - False if not
"""
server = self._get_horizon_server()
endpoint = server.operations().for_transaction(transaction_hash)
response = endpoint.call()
results = response["_embedded"]["records"][0]
return results["type"] == "payment"
def get_asset(self, code="TFT", issuer=None) -> stellar_sdk.Asset:
"""Gets an stellar_sdk.Asset object by code.
if the code is TFT or TFTA we quickly return the Asset object based on the code.
if the code is native (XLM) we return the Asset object with None issuer.
if the code isn't unknown, exception is raised to manually construct the Asset object.
Args:
code (str, optional): code for the asset. Defaults to "TFT".
issuer (str, optional): issuer for the asset. Defaults to None.
Raises:
ValueError: empty code, In case of issuer is None and not XLM or the code isn't for TFT or TFTA.
stellar_sdk.exceptions.AssetIssuerInvalidError: Invalid issuer
Returns:
stellar_sdk.Asset: Asset object.
"""
network = self.network.value
KNOWN_ASSETS = list(_NETWORK_KNOWN_TRUSTS[network].keys()) + ["XLM"]
if issuer and code:
return Asset(code, issuer)
if not code:
raise ValueError("need to provide code")
if not issuer and code not in KNOWN_ASSETS:
raise ValueError(
f"Make sure to supply the issuer for {code}, issuer is allowed to be none only in case of {KNOWN_ASSETS}"
)
if not issuer and code in KNOWN_ASSETS:
asset_issuer = _NETWORK_KNOWN_TRUSTS[network].get(code, None)
return Asset(code, asset_issuer)
def cancel_sell_order(
self,
offer_id,
selling_asset: stellar_sdk.Asset,
buying_asset: stellar_sdk.Asset,
price: Union[str, decimal.Decimal],
):
"""Deletes a selling order for amount `amount` of `selling_asset` for `buying_asset` with the price of `price`
Args:
selling_asset (stellar_sdk.Asset): Selling Asset object - check wallet object.get_asset_by_code function
buying_asset (stellar_sdk.Asset): Buying Asset object - Asset object - check wallet object.get_asset_by_code function
offer_id (int): pass the current offer id and set the amount to 0 to cancel this offer
price (str): order price
"""
return self._manage_sell_order(
selling_asset=selling_asset, buying_asset=buying_asset, amount="0", price=price, offer_id=offer_id
)
def _manage_sell_order(
self,
selling_asset: stellar_sdk.Asset,
buying_asset: stellar_sdk.Asset,
amount: Union[str, decimal.Decimal],
price: Union[str, decimal.Decimal],
timeout=30,
offer_id=0,
):
"""Places/Deletes a selling order for amount `amount` of `selling_asset` for `buying_asset` with the price of `price`
Args:
selling_asset (stellar_sdk.Asset): Selling Asset object - check wallet object.get_asset_by_code function
buying_asset (stellar_sdk.Asset): Buying Asset object - Asset object - check wallet object.get_asset_by_code function
amount (Union[str, decimal.Decimal]): Amount to sell.
price (Union[str, decimal.Decimal]): Price for selling.
timeout (int, optional): Timeout for submitting the transaction. Defaults to 30.
offer_id: pass the current offer id and set the amount to 0 to cancel this offer or another amount to update the offer
Raises:
ValueError: In case of invalid issuer.
RuntimeError: Error happened during submission of the transaction.
Returns:
(dict): response as the result of sumbit the transaction
"""
server = self._get_horizon_server()
tb = TransactionBuilder(self.load_account(), network_passphrase=_NETWORK_PASSPHRASES[self.network.value])
try:
tx = (
tb.append_manage_sell_offer_op(
selling_code=selling_asset.code,
selling_issuer=selling_asset.issuer,
buying_code=buying_asset.code,
buying_issuer=buying_asset.issuer,
amount=amount,
price=price,
offer_id=offer_id,
)
.set_timeout(timeout)
.build()
)
except stellar_sdk.exceptions.AssetIssuerInvalidError as e:
raise ValueError("invalid issuer") from e
except Exception as e:
raise RuntimeError(
f"error happened for placing selling order for selling: {selling_asset}, buying: {buying_asset}, amount: {amount} price: {price}"
) from e
else:
tx.sign(self.secret)
try:
resp = server.submit_transaction(tx)
except Exception as e:
raise RuntimeError(
f"couldn't submit sell offer, probably wallet is unfunded. Please check the error stacktrace for more information."
) from e
return resp
place_sell_order = _manage_sell_order
def get_created_offers(self, wallet_address: str = None):
"""Returns a list of the currently created offers
Args:
wallet_address (Str, optional): wallet address you want to get offers to. Defaults to self.address.
Returns:
list
"""
wallet_address = wallet_address or self.address
server = self._get_horizon_server()
endpoint = server.offers()
endpoint.account(wallet_address)
response = endpoint.call()
offers = response["_embedded"]["records"]
return offers
def set_data_entry(self, name: str, value: str, address: str = None):
"""Sets, modifies or deletes a data entry (name/value pair) for an account
To delete a data entry, set the value to an empty string.
"""
address = address or self.address
signing_key = stellar_sdk.Keypair.from_secret(self.secret)
horizon_server = self._get_horizon_server()
if address == self.address:
account = self.load_account()
else:
account = horizon_server.load_account(address)
base_fee = horizon_server.fetch_base_fee()
transaction = (
stellar_sdk.TransactionBuilder(
source_account=account, network_passphrase=_NETWORK_PASSPHRASES[self.network.value], base_fee=base_fee,
)
.append_manage_data_op(name, value)
.set_timeout(30)
.build()
)
transaction.sign(signing_key)
try:
response = horizon_server.submit_transaction(transaction)
j.logger.info("Transaction hash: {}".format(response["hash"]))
except stellar_sdk.exceptions.BadRequestError as e:
j.logger.debug(e)
raise e
def get_data_entries(self, address: str = None):
address = address or self.address
horizon_server = self._get_horizon_server()
response = horizon_server.accounts().account_id(address).call()
data = {}
for data_name, data_value in response["data"].items():
data[data_name] = base64.b64decode(data_value).decode("utf-8")
return data
class User(Base):
emails = fields.List(fields.String())
permissions = fields.List(fields.Object(Permission))
custom_config = fields.Typed(dict)
type = fields.Enum(UserType)
password = fields.Secret()
class ZDBClient(Client):
addr = fields.String(default="localhost")
port = fields.Integer(default=9900)
secret_ = fields.String(default="1234567")
nsname = fields.String(default="test")
admin = fields.Boolean(default=False)
mode = fields.Enum(Mode)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# if not self.secret_:
# self.secret_ = j.core.myenv.adminsecret
assert len(self.secret_) > 5
if self.admin:
self.nsname = "default"
self.type = "ZDB"
self._redis = None
self.nsname = self.nsname.lower().strip()
# if j.data.bcdb._master:
# self._model.trigger_add(self._update_trigger)
def _update_trigger(self, obj, action, **kwargs):
if action in ["save", "change"]:
self._redis = None
@property
def redis(self):
if not self._redis:
pool = redis.ConnectionPool(
host=self.addr,
port=self.port,
password=self.secret_,
connection_class=ZDBConnection,
namespace=self.nsname,
namespace_password=self.secret_,
admin=self.admin,
)
self._redis = _patch_redis_client(redis.Redis(connection_pool=pool))
return self._redis
def _key_encode(self, key):
if self.mode.value == Mode.SEQ.value:
if key is None:
key = ""
else:
key = struct.pack("<I", key)
return key
def _key_decode(self, key):
if self.mode.value == Mode.SEQ.value:
key = struct.unpack("<I", key)[0]
return key
def set(self, data, key=None):
key = key or ""
key = self._key_encode(key)
res = self.redis.execute_command("SET", key, data)
if not res:
return res
return self._key_decode(res)
def get(self, key):
key = self._key_encode(key)
return self.redis.execute_command("GET", key)
def exists(self, key):
key = self._key_encode(key)
return self.redis.execute_command("EXISTS", key) == 1
def delete(self, key):
if not key:
raise j.exceptions.Value("key must be provided")
key = self._key_encode(key)
self.redis.execute_command("DEL", key)
def flush(self):
"""
will remove all data from the database DANGEROUS !!!!
This is only allowed on private and password protected namespace
You need to select the namespace before running the command.
:return:
"""
if not self.nsname in ["default", "system"]:
self.redis.execute_command("FLUSH")
def stop(self):
pass
@property
def nsinfo(self):
cmd = self.redis.execute_command("NSINFO", self.nsname)
return _parse_nsinfo(cmd.decode())
def list(self, key_start=None, reverse=False):
"""
list all the keys in the namespace
:param key_start: if specified start to walk from that key instead of the first one, defaults to None
:param key_start: str, optional
:param reverse: decide how to walk the namespace
if False, walk from older to newer keys
if True walk from newer to older keys
defaults to False
:param reverse: bool, optional
:return: list of keys
:rtype: [str]
"""
result = []
for key, data in self.iterate(key_start=key_start, reverse=reverse, keyonly=True):
result.append(key)
return result
def iterate(self, key_start=None, reverse=False, keyonly=False):
"""
walk over all the namespace and yield (key,data) for each entries in a namespace
:param key_start: if specified start to walk from that key instead of the first one, defaults to None
:param key_start: str, optional
:param reverse: decide how to walk the namespace
if False, walk from older to newer keys
if True walk from newer to older keys
defaults to False
:param reverse: bool, optional
:param keyonly: [description], defaults to False
:param keyonly: bool, optional
:raises e: [description]
"""
next = None
data = None
if key_start is not None:
next = self.redis.execute_command("KEYCUR", self._key_encode(key_start))
if not keyonly:
data = self.get(key_start)
yield (key_start, data)
CMD = "SCANX" if not reverse else "RSCAN"
while True:
try:
if not next:
response = self.redis.execute_command(CMD)
else:
response = self.redis.execute_command(CMD, next)
# format of the response
# see https://github.com/threefoldtech/0-db/tree/development#scan
except redis.ResponseError as e:
if e.args[0] == "No more data":
return
raise e
(next, results) = response
for item in results:
keyb, size, epoch = item
key_new = self._key_decode(keyb)
data = None
if not keyonly:
data = self.redis.execute_command("GET", keyb)
yield (key_new, data)
@property
def count(self):
"""
:return: return the number of entries in the namespace
:rtype: int
"""
return self.nsinfo["entries"]
def ping(self):
"""
go to default namespace & ping
:return:
"""
return self.redis.ping()
@property
def next_id(self):
"""
:return: return the next id
:rtype: int
"""
id_bytes = struct.pack("<I", int(self.nsinfo["next_internal_id"], 16))
return int.from_bytes(id_bytes, byteorder="big", signed=True)
class OpenRestyServer(Base):
status = fields.Enum(Status)
websites = fields.Factory(Website)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._cmd = None
self._path_web = None
self._path_cfg_dir = None
self._logs_dir = None
self.executor = "tmux" # only tmux for now
@property
def path_web(self):
if not self._path_web:
self._path_web = j.sals.fs.join_paths(j.core.dirs.VARDIR, "web",
self.instance_name)
j.sals.fs.mkdirs(j.sals.fs.join_paths(self._path_web, "static"))
return self._path_web
@property
def path_cfg_dir(self):
if not self._path_cfg_dir:
self._path_cfg_dir = j.sals.fs.join_paths(j.core.dirs.CFGDIR,
"nginx",
self.instance_name)
j.sals.fs.mkdirs(self._path_cfg_dir)
return self._path_cfg_dir
@property
def path_cfg(self):
return j.sals.fs.join_paths(self.path_cfg_dir, "nginx.conf")
@property
def logs_dir(self):
if not self._logs_dir:
self._logs_dir = j.sals.fs.join_paths(j.core.dirs.LOGDIR,
"openresty",
self.instance_name)
j.sals.fs.mkdirs(self._logs_dir)
return self._logs_dir
def configure(self):
# clean old websites config
self.cleanup()
"""configures main nginx conf
"""
# self.install() This is commented for now until the repo and necessary deps are handled
configtext = j.tools.jinja2.render_template(
template_path=j.sals.fs.join_paths(DIR_PATH, "templates",
"nginx.conf"),
logs_dir=self.logs_dir,
)
j.sals.fs.write_file(self.path_cfg, configtext)
def get_from_port(self, port, domain=None, ssl=None):
"""will try to get a website listening on port, if it doesn't exist it will create one
Args:
port (int): port to search for
domain (str, optional): domain. Defaults to None.
ssl (bool, optional): Will set ssl if True. Defaults to None.
Returns:
Website: A new or an old website instance with the needed port
"""
website_name = f"{self.instance_name}_website_{port}"
website = self.websites.find(website_name)
if website:
return website
website = self.websites.get(website_name)
ssl = ssl or port == 443 # Use ssl if port is 443 if ssl in not specified
website.domain = domain
website.port = port
website.ssl = ssl
return website
def install(self, reset=False):
"""Install required deps for openresty
Args:
reset (bool, optional): If true will redo the installation. Defaults to False.
"""
if reset or self.status == "init":
# get weblib
weblibs_path = j.tools.git.ensure_repo(
"https://github.com/threefoldtech/js-weblibs" # Place holder repo might be changed
)
# copy the templates to the right location
j.sals.fs.copy_tree(f"{DIR_PATH}/web_resources/",
self.path_cfg_dir)
j.sals.fs.symlink(
f"{weblibs_path}/static",
f"{self.path_web}/static/weblibs",
overwrite=True,
)
self.status = Status.INSTALLED
self.save()
@property
def startup_cmd(self):
pass
def start(self, reset=False):
pass
def stop(self):
pass
def is_running(self):
pass
def reload(self):
self.configure()
j.sals.process.execute("lapis build", cwd=self.path_cfg_dir)
def cleanup(self):
j.sals.fs.rmtree(f"{self.path_cfg_dir}/servers")