def test_url(self):
"""Success scenario for URL validation"""
url = fields.URL("https://www.test.com")
self.assertEqual(url.default, "https://www.test.com")
self.assertIsNone(url.validate("https://www.test.com"))
"""Failure scenario for URL validation"""
url = fields.URL("https://www.test.com")
self.assertEqual(url.default, "https://www.test.com")
with self.assertRaises(ValidationError):
url.validate("test")
class ZerosslCertbot(NginxCertbot):
SERVER_URL = "https://acme.zerossl.com/v2/DV90"
KEY_CREDENTIALS_URL = "https://api.zerossl.com/acme/eab-credentials"
EMAIL_CREDENTIALS_URL = "https://api.zerossl.com/acme/eab-credentials-email"
api_key_ = fields.Secret()
server = fields.URL(default=SERVER_URL)
@property
def run_cmd(self):
# get eab_kid and eab_hmac_key based on email or api_key_
if not self.email and not self.api_key_:
raise Input("email or api_key_ must be provided")
# set them to get the full run-cmd with correct arguments
if self.api_key_:
resp = j.tools.http.post(self.KEY_CREDENTIALS_URL, params={"access_key": self.api_key_})
else:
resp = j.tools.http.post(self.EMAIL_CREDENTIALS_URL, data={"email": self.email})
resp.raise_for_status()
data = resp.json()
self.eab_kid = data["eab_kid"]
self.eab_hmac_key = data["eab_hmac_key"]
return super().run_cmd
class Student(Base):
ID = fields.Integer()
name = fields.String()
email = fields.Email()
tel = fields.Tel()
web = fields.URL()
birthday = fields.Date()
class Wallet(Base):
ID = fields.Integer(required=True)
origin = fields.Typed(dict, default=dict)
addresses = fields.Factory(Address)
key = fields.Bytes()
email = fields.Email()
url = fields.URL(required=False, allow_empty=True)
data = fields.Json(allow_empty=False)
class Certbot(Base):
DEFAULT_NAME = "certbot"
DEFAULT_LOGS_DIR = j.sals.fs.join_paths(j.core.dirs.LOGDIR, DEFAULT_NAME)
DEFAULT_CONFIG_DIR = j.sals.fs.join_paths(j.core.dirs.CFGDIR, DEFAULT_NAME)
DEFAULT_WORK_DIR = j.sals.fs.join_paths(j.core.dirs.VARDIR, DEFAULT_NAME)
# the following options match the certbot command arguments
domain = fields.String(required=True)
non_interactive = fields.Boolean(default=True)
agree_tos = fields.Boolean(default=True)
logs_dir = fields.String(default=DEFAULT_LOGS_DIR)
config_dir = fields.String(default=DEFAULT_CONFIG_DIR)
work_dir = fields.String(default=DEFAULT_WORK_DIR)
email = fields.Email()
server = fields.URL()
eab_kid = fields.String()
eab_hmac_key = fields.String()
# for existing certificates
key_path = fields.String()
cert_path = fields.String()
fullchain_path = fields.String()
@property
def run_cmd(self):
args = [self.DEFAULT_NAME]
for name, value in self.to_dict().items():
if name.endswith("_"):
continue
if value:
# append only if the field has a value
name = name.replace("_", "-")
args.append(f"--{name}")
# append the value itself only if it's a boolean value
# boolean options are set by adding name only
if not isinstance(value, bool):
args.append(value)
return args
@property
def install_cmd(self):
# replace "certbot" with "certbot install"
cmd = self.run_cmd
cmd.insert(1, "install")
return cmd
@property
def renew_cmd(self):
# replace "certbot" with "certbot install"
renew_certbot = Certbot(work_dir=self.work_dir, config_dir=self.config_dir, logs_dir=self.logs_dir, domain="")
cmd = renew_certbot.run_cmd
cmd.insert(1, "renew")
return cmd
class ThreebotServer(Base):
_package_manager = fields.Factory(PackageManager)
domain = fields.String()
email = fields.String()
acme_server_type = fields.Enum(AcmeServer)
acme_server_url = fields.URL()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._rack = None
self._gedis = None
self._db = None
self._gedis_http = None
self._services = None
self._packages = None
self._started = False
self._nginx = None
self._redis = None
self.rack.add(GEDIS, self.gedis)
self.rack.add(GEDIS_HTTP, self.gedis_http.gevent_server)
self.rack.add(SERVICE_MANAGER, self.services)
def is_running(self):
nginx_running = self.nginx.is_running()
redis_running = self.redis.cmd.is_running(
) or j.sals.nettools.wait_connection_test("127.0.0.1", 6379, timeout=1)
gedis_running = j.sals.nettools.wait_connection_test("127.0.0.1",
16000,
timeout=1)
return nginx_running and redis_running and gedis_running
@property
def started(self):
return self._started
@property
def nginx(self):
if self._nginx is None:
self._nginx = j.tools.nginx.get("default")
return self._nginx
@property
def redis(self):
if self._redis is None:
self._redis = j.tools.redis.get("default")
return self._redis
@property
def db(self):
if self._db is None:
self._db = j.core.db
return self._db
@property
def rack(self):
if self._rack is None:
self._rack = j.servers.rack
return self._rack
@property
def gedis(self):
if self._gedis is None:
self._gedis = j.servers.gedis.get("threebot")
return self._gedis
@property
def gedis_http(self):
if self._gedis_http is None:
self._gedis_http = j.servers.gedis_http.get("threebot")
return self._gedis_http
@property
def services(self):
if self._services is None:
self._services = j.tools.servicemanager.get("threebot")
return self._services
@property
def chatbot(self):
return self.gedis._loaded_actors.get("chatflows_chatbot")
@property
def packages(self):
if self._packages is None:
self._packages = self._package_manager.get(self.instance_name)
return self._packages
def check_dependencies(self):
install_msg = "Visit https://github.com/threefoldtech/js-sdk/blob/development/docs/wiki/quick_start.md for installation guide"
if not self.nginx.installed:
raise j.exceptions.NotFound(
f"nginx is not installed.\n{install_msg}")
ret = shutil.which("certbot")
if not ret:
raise j.exceptions.NotFound(
f"certbot is not installed.\n{install_msg}")
rc, out, err = j.sals.process.execute("certbot plugins")
if "* nginx" not in out:
raise j.exceptions.NotFound(
f"python-certbot-nginx is not installed.\n{install_msg}")
if not self.redis.installed:
raise j.exceptions.NotFound(
f"redis is not installed.\n{install_msg}")
ret = shutil.which("tmux")
if not ret:
raise j.exceptions.NotFound(
f"tmux is not installed.\n{install_msg}")
ret = shutil.which("git")
if not ret:
raise j.exceptions.NotFound(
f"git is not installed.\n{install_msg}")
def start(self, wait: bool = False):
# start default servers in the rack
# handle signals
for signal_type in (signal.SIGTERM, signal.SIGINT, signal.SIGKILL):
gevent.signal(signal_type, self.stop)
# mark app as started
if self.is_running():
return
self.check_dependencies()
self.redis.start()
self.nginx.start()
self.rack.start()
j.logger.register(f"threebot_{self.instance_name}")
# add default packages
for package_name in DEFAULT_PACKAGES:
j.logger.info(f"Configuring package {package_name}")
try:
package = self.packages.get(package_name)
self.packages.install(package)
except Exception as e:
self.stop()
raise j.core.exceptions.Runtime(
f"Error happened during getting or installing {package_name} package, the detailed error is {str(e)}"
) from e
# install all package
self.packages._install_all()
j.logger.info("Reloading nginx")
self.nginx.reload()
# mark server as started
self._started = True
j.logger.info(
f"Threebot is running at http://localhost:{PORTS.HTTP} and https://localhost:{PORTS.HTTPS}"
)
self.rack.start(wait=wait) # to keep the server running
def stop(self):
server_packages = self.packages.list_all()
for package_name in server_packages:
package = self.packages.get(package_name)
package.stop()
self.nginx.stop()
# mark app as stopped, do this before stopping redis
j.logger.unregister()
self.redis.stop()
self.rack.stop()
self._started = False
class Website(Base):
domain = fields.String()
ssl = fields.Boolean()
port = fields.Integer(default=PORTS.HTTP)
locations = fields.Factory(Location, stored=False)
includes = fields.List(fields.String())
selfsigned = fields.Boolean(default=True)
# keep it as letsencryptemail for compatibility
letsencryptemail = fields.String()
acme_server_type = fields.Enum(AcmeServer)
acme_server_url = fields.URL()
# in case of using existing key/certificate
key_path = fields.String()
cert_path = fields.String()
fullchain_path = fields.String()
@property
def certbot(self):
kwargs = dict(
domain=self.domain,
email=self.letsencryptemail,
server=self.acme_server_url,
nginx_server_root=self.parent.cfg_dir,
key_path=self.key_path,
cert_path=self.cert_path,
fullchain_path=self.fullchain_path,
)
if self.acme_server_type == AcmeServer.LETSENCRYPT:
certbot_type = LetsencryptCertbot
elif self.acme_server_type == AcmeServer.ZEROSSL:
certbot_type = ZerosslCertbot
else:
certbot_type = CustomCertbot
return certbot_type(**kwargs)
@property
def cfg_dir(self):
return j.sals.fs.join_paths(self.parent.cfg_dir, self.instance_name)
@property
def cfg_file(self):
return j.sals.fs.join_paths(self.cfg_dir, "server.conf")
@property
def include_paths(self):
paths = []
for include in self.includes:
## TODO validate location name and include
website_name, location_name = include.split(".", 1)
website = self.parent.websites.find(website_name)
if not website:
continue
paths.append(j.sals.fs.join_paths(website.cfg_dir, "locations", location_name))
return paths
def get_locations(self):
for location in self.locations.list_all():
yield self.locations.get(location)
def get_proxy_location(self, name):
location = self.locations.get(name)
location.location_type = LocationType.PROXY
return location
def get_custom_location(self, name):
location = self.locations.get(name)
location.location_type = LocationType.CUSTOM
return location
def get_static_location(self, name):
location = self.locations.get(name)
location.location_type = LocationType.STATIC
return location
def get_config(self):
return render_config_template("website", base_dir=j.core.dirs.BASEDIR, website=self)
def generate_certificates(self, retries=6):
if self.domain:
if self.key_path and self.cert_path and self.fullchain_path:
# only use install command if an existing key and certificate were set
self.install_certifcate()
else:
self.obtain_and_install_certifcate(retries=retries)
def install_certifcate(self):
"""Construct and Execute install certificate command
Alternative to certbot install
"""
cmd = self.certbot.install_cmd
j.logger.debug(f"Execute: {' '.join(cmd)}")
rc, out, err = j.sals.process.execute(cmd)
if rc > 0:
j.logger.error(f"Installing certificate failed {out}\n{err}")
else:
j.logger.info(f"Certificate installed successfully {out}")
def obtain_and_install_certifcate(self, retries=6):
"""Construct and Execute run certificate command,This will issue a new certificate managed by Certbot
Alternative to certbot run
Args:
retries (int, optional): Number of retries Certbot will try to install the certificate if failed. Defaults to 6.
"""
cmd = self.certbot.run_cmd
j.logger.debug(f"Execute: {' '.join(cmd)}")
for _ in range(retries):
rc, out, err = j.sals.process.execute(cmd)
if rc > 0:
j.logger.error(f"Generating certificate failed {out}\n{err}")
else:
j.logger.error(f"Certificate Generated successfully {out}")
break
def generate_self_signed_certificates(self):
keypempath = f"{self.parent.cfg_dir}/key.pem"
certpempath = f"{self.parent.cfg_dir}/cert.pem"
if j.sals.process.is_installed("mkcert"):
res = j.sals.process.execute(
f"mkcert -key-file {keypempath} -cert-file {certpempath} localhost *.localhost 127.0.0.1 ::1"
)
if res[0] != 0:
raise j.exceptions.JSException(f"Failed to generate self-signed certificate (using mkcert).{res}")
else:
if j.sals.fs.exists(f"{keypempath}") and j.sals.fs.exists(f"{certpempath}"):
return
res = j.sals.process.execute(
f"openssl req -nodes -x509 -newkey rsa:4096 -keyout {keypempath} -out {certpempath} -days 365 -subj '/CN=localhost'"
)
if res[0] != 0:
raise j.exceptions.JSException(f"Failed to generate self-signed certificate (using openssl).{res}")
def configure(self, generate_certificates=True):
j.sals.fs.mkdir(self.cfg_dir)
needed_dirs = ("body", "client-body", "fastcgi", "proxy", "scgi", "uwsgi")
for d in needed_dirs:
j.sals.fs.mkdir(j.sals.fs.join_paths(self.cfg_dir, d))
for location in self.get_locations():
location.configure()
j.sals.fs.write_file(self.cfg_file, self.get_config())
if self.ssl:
self.generate_self_signed_certificates()
if generate_certificates and self.ssl:
self.generate_certificates()
def clean(self):
j.sals.fs.rmtree(self.cfg_dir)
class CustomCertbot(NginxCertbot):
# change email and server required value to True here
email = fields.Email(required=True)
server = fields.URL(required=True)
