def run(self):
# behold the mighty worker-loop
while True:
pending_jobs = []
try:
job = self.my_job_queue.get(timeout=1)
if isinstance(job, str):
# handle job w/o args, comes as `name: str`
pending_jobs.append((job, None))
else:
# already comes as `(name: str, args: tupel)`
pending_jobs.append(job)
except Empty:
# returns: list[job: tuple[str, tuple]]
pending_jobs = self.job_mgr.get_recurring_jobs()
# nothing to do => power nap
if len(pending_jobs) == 0:
sleep(0.5)
continue
for job_name, job_args in pending_jobs:
# special job "exit" will stop the worker-queue
if job_name == "exit":
log.info("exiting (background worker) by request")
return
# finally handle (dispatch): `job_name` + `job_args`
self.job_mgr.handle_job(job_name, job_args)
def import_config_dir(self, src_path):
# regulary copy over all files
self.import_dir("config", src_path, block=True)
log.info("copied configs done - updating db-password")
# afterwards make sure the password for sql is correct inside config.php
cfg_path = Path(self.dirs["config"]) / "config.php"
cfg_content = cfg_path.read_text()
# get pass from docker.env
db_env_dct = self.get_env_data()
dbpass = db_env_dct["MYSQL_PASSWORD"]
# read config, regex dbpassword, replace with new pass (from docker.env)
pat = re.compile(r"'dbpassword'[^=]*=>[^']*'([^']*)'[^,]*,")
res = pat.search(cfg_content)
old_pass = res.group()
new_pass = f"'dbpassword' => '{dbpass}',"
with cfg_path.open("w") as fd:
fd.write(cfg_content.replace(old_pass, new_pass))
log.info("updated config.php with 'nextcloud' db password")
return True
def register_job(self, job):
"""register job by `job.name` and construct their instances"""
if job.name in self.jobs:
log.warning(f"overwriting job (during register) with name: {job.name}")
self.jobs[job.name] = job()
log.info(f"registered job: {job.name}")
def export_dir(self, key, tar_dir):
self.activity_desc = (key, "export")
self.rsync_dir(key, self.dirs[key], tar_dir)
log.info(f"starting {key} export")
if self.rsync_stats:
self.update_meta(tar_dir,
{"size_" + key: self.rsync_stats.get("size", 0)})
def import_dir(self, key, src_dir, block=False):
self.activity_desc = (key, "import")
src_dir = Path(src_dir)
key_path = Path(self.dirs[key])
src_path = src_dir / key_path.name
tar_path = key_path.parent
self.rsync_dir("data",
src_path.as_posix(),
tar_path.as_posix(),
block_main=block)
log.info(f"starting {key} import")
def _run(self, cfg, board, kwargs):
# only enable, if nextcloud is installed
if not self.nc.is_installed:
log.debug("cannot enable nextbox-app - uninitialized nextcloud")
return
# try to enable
try:
if self.nc.enable_nextbox_app():
self.interval = 3600
log.info("enabled nextcloud nextbox-app")
except NextcloudError:
pass
def restore_start():
src_path = request.form.get("src_path")
if not backup_restore.check_backup(src_path):
msg = "Invalid backup, cannot restore"
log.error(msg)
return error(msg)
log.info(f"Initiating restore from: {src_path}")
job_kwargs = {"tar_path": src_path, "mode": "restore"}
job_queue.put(("BackupRestore", job_kwargs))
return success("restore started")
def setup(self, token, subdomain, scheme):
if scheme not in ["http", "https"]:
raise ProxySetupError(
f"provided scheme: {scheme} not 'http' or 'https'")
local_port = 80 if scheme == "http" else 443
remote_port = self.register_at_server(token, subdomain, scheme)
self.create_config(token, remote_port, local_port)
self.start()
log.info("setup proxy:")
log.info(f"port: {local_port}, remote_port: {remote_port} @ {scheme}")
return remote_port
def ensure_apache_mods(self):
avail = Path(self.apache_mods_available)
enabled = Path(self.apache_mods_enabled)
for mod in self.apache_mods:
conf_avail_path = (avail / (mod + ".conf"))
load_avail_path = (avail / (mod + ".load"))
conf_enabled_path = (enabled / (mod + ".conf"))
load_enabled_path = (enabled / (mod + ".load"))
if conf_avail_path.exists() and not conf_enabled_path.exists():
conf_enabled_path.symlink_to("../mods-available/" +
conf_avail_path.name)
log.info(f"enabled: {conf_avail_path.name}")
if load_avail_path.exists() and not load_enabled_path.exists():
load_enabled_path.symlink_to("../mods-available/" +
load_avail_path.name)
log.info(f"enabled: {load_avail_path.name}")
def _run(self, cfg, board, kwargs):
token = cfg["config"]["desec_token"]
dns_mode = cfg["config"]["dns_mode"]
domain = cfg["config"]["domain"]
if dns_mode not in ["desec_done", "off"]:
if not services.is_active("ddclient"):
services.restart("ddclient")
services.enable("ddclient")
return
if dns_mode == "off":
if services.is_active("ddclient"):
services.stop("ddclient")
services.disable("ddclient")
return
if dns_mode == "desec_done":
if services.is_active("ddclient"):
services.stop("ddclient")
services.disable("ddclient")
dns = DNSManager()
ipv4 = dns.get_ipv4()
ipv6 = dns.get_ipv6()
headers = {"Authorization": f"Token {token}"}
params = {"hostname": domain}
if ipv4:
params["myipv4"] = ipv4
if ipv6:
params["myipv6"] = ipv6
res = requests.get("https://update.dedyn.io", params=params, headers=headers)
if res.ok:
log.info(f"updated deSEC IPv4 ({ipv4}) and IPv6 ({ipv6}) address for '{domain}'")
else:
log.warning(f"failed updating IPs ({ipv4} & {ipv6}) for domain: '{domain}'")
log.debug(f"result: {res.text} status_code: {res.status_code} url: {res.url}")
def _run(self, cfg, board, kwargs):
self.interval = None
# if the custom-dns-config is active, ensure ddclient is up
if cfg["config"]["dns_mode"] == "config_done" and not services.is_active("ddclient"):
services.restart("ddclient")
# ensure that neither updater nor factory-reset are masked (paranoia!)
services.unmask("nextbox-updater")
services.unmask("nextbox-factory-reset")
# log a welcome + version
log.info(f"Hello World - I am NextBox - call me: v{nextbox_version()} - let'sa gooo")
# update/upgrade now
shield.set_led_state("updating")
log.info("running 'apt-get update'")
cache = apt.cache.Cache()
cache.update()
cache.open()
# which debian package
pkg = cfg["config"]["debian_package"]
try:
pkg_obj = cache[pkg]
except KeyError:
log.error(f"self-update failed: designated package: {pkg} not found!")
log.error("falling back to 'nextbox' - retrying upgrade...")
pkg = "nextbox"
try:
pkg_obj = cache[pkg]
except KeyError:
log.error("CRITICAL: failed to find 'nextbox' in apt-cache")
# we should never ever end here, this means that the nextbox
# debian package is not available...
# nextbox debian (ppa) repository not available ???!!
return
# install package (i.e., other nextbox package is already installed)
# will trigger for e.g., 'nextbox' to 'nextbox-testing' switching
if not pkg_obj.is_installed:
log.info(f"installing debian package: {pkg} (start service: nextbox-updater)")
services.start("nextbox-updater")
elif pkg_obj.is_upgradable:
log.info(f"upgrading debian package: {pkg} (start service: nextbox-updater)")
services.start("nextbox-updater")
else:
log.debug(f"no need to upgrade or install debian package: {pkg}")
def backup_start():
tar_path = request.form.get("tar_path")
found = False
for dev in partitions.backup_devices:
if tar_path.startswith(dev["path"]):
found = dev
break
if not found:
msg = "Invalid backup location provided"
log.error(msg)
return error(msg)
log.info(
f"Initiating backup onto: {dev['friendly_name']} @ {dev['path']} with target: {tar_path}"
)
job_kwargs = {"tar_path": tar_path, "mode": "backup"}
job_queue.put(("BackupRestore", job_kwargs))
return success("backup started")
def ssh_set():
auth_p_dir = Path("/home/nextuser/.ssh")
if not auth_p_dir.exists():
os.makedirs(auth_p_dir.as_posix())
auth_p = auth_p_dir / "authorized_keys"
if request.method == "GET":
ip = local_ip()
if not auth_p.exists():
return success(data={"pubkey": "", "local_ip": ip})
with auth_p.open() as fd:
return success(data={"pubkey": fd.read(), "local_ip": ip})
elif request.method == "POST":
pubkey = request.form.get("pubkey")
with auth_p.open("w") as fd:
fd.write(pubkey.strip() + "\n")
log.info(f"setting ssh pub key: {pubkey}")
return success()
def test_resolve4():
ip_type = request.path.split("/")[-1]
domain = cfg["config"]["domain"]
resolve_ip = None
ext_ip = None
if not domain:
return error("no domain is set")
dns = DNSManager()
dns.clear_dns_caches()
if ip_type == "ipv4":
resolve_ip = dns.resolve_ipv4(domain)
ext_ip = dns.get_ipv4()
else:
resolve_ip = dns.resolve_ipv6(domain)
ext_ip = dns.get_ipv6()
log.info(
f"resolving '{domain}' to IP: {resolve_ip}, external IP: {ext_ip}")
data = {"ip": ext_ip, "resolve_ip": resolve_ip, "domain": domain}
# if not both "resolve" and "getip" are successful, we have failed
if resolve_ip is None or ext_ip is None:
log.error(f"failed resolving and/or getting external {ip_type}")
return error("Resolve test: Not OK", data=data)
# resolving to wrong ip
if resolve_ip != ext_ip:
log.warning(f"Resolved {ip_type} does not match external {ip_type}")
log.warning("This might indicate a bad DynDNS configuration")
return error("Resolve test: Not OK", data=data)
# all good!
return success("Resolve test: OK", data=data)
def signal_handler(sig, frame):
global job_queue, worker
log.info("signal-handler, sending 'exit' as job for graceful termination")
job_queue.put("exit")
worker.join()
log.info("joined background-worker - exiting now...")
log.info("^" * 60)
sys.exit(1)
def reboot():
log.info("REBOOTING NOW - by /reboot request")
cr = CommandRunner("reboot")
if cr.returncode != 0:
return error("failed executing: 'reboot'")
return success(data={})
def _run(self, cfg, board, kwargs):
board.set("tls", {
"what": "enable",
"state": "running",
})
certs = Certificates()
domain = cfg.get("config", {}).get("domain")
email = cfg.get("config", {}).get("email")
if not domain or not email:
log.error(f"cannot enable tls, domain: '{domain}' email: '{email}'")
board.set("tls", {
"what": "domain-or-email",
"state": "fail",
})
return False
my_cert = certs.get_cert(domain)
if not my_cert:
log.warning(f"could not get local certificate for: {domain}, acquiring...")
if not certs.acquire_cert(domain, email):
msg = f"Failed to acquire {domain} with {email}"
log.error(msg)
board.set("tls", {
"what": "acquire",
"state": "fail",
})
return False
log.info(f"acquired certificate for: {domain} with {email}")
my_cert = certs.get_cert(domain)
certs.write_apache_ssl_conf(
my_cert["domains"][0],
my_cert["fullchain_path"],
my_cert["privkey_path"]
)
if not certs.set_apache_config(ssl=True):
board.set("tls", {
"what": "apache-config",
"state": "fail",
})
log.error("failed enabling ssl configuration for apache")
return False
log.info(f"activated https for apache using: {domain}")
cfg["config"]["https_port"] = 443
cfg["config"]["email"] = email
cfg.save()
board.set("tls", {
"what": "enable",
"state": "success",
})
# we need to "re-wire" the proxy to port 443 on activated TLS
add_msg = ""
if cfg["config"]["proxy_active"]:
subdomain = cfg["config"]["proxy_domain"].split(".")[0]
scheme = "https"
token = cfg["config"]["nk_token"]
proxy_tunnel = ProxyTunnel()
try:
proxy_port = proxy_tunnel.setup(token, subdomain, scheme)
except ProxySetupError as e:
log.error("register at proxy-server error", exc_info=e)
except Exception as e:
log.error("unexpected error during proxy setup", exc_info=e)
cfg["config"]["proxy_port"] = proxy_port
cfg.save()
def poweroff():
log.info("POWER-OFF - by /poweroff request")
cr = CommandRunner("poweroff")
if cr.returncode != 0:
return error("failed executing: 'poweroff'")
return success(data={})
def import_sql(self, src_path):
db_env_dct = self.get_env_data()
pwd = db_env_dct["MYSQL_ROOT_PASSWORD"]
if pwd is None:
log.error("cannot get (root) sql password for import, aborting...")
return False
src_sql_path = Path(src_path) / self.sql_dump_fn
if not src_sql_path.exists():
log.error("sql-import data path not found, aborting...")
return False
# drop (new) database
cmd = self.db_cmd.format(pwd=pwd,
sql="DROP DATABASE IF EXISTS new_nextcloud")
cr = CommandRunner.retries(5, cmd, block=True)
# create new database
cmd = self.db_cmd.format(pwd=pwd, sql="CREATE DATABASE new_nextcloud")
cr = CommandRunner.retries(5, cmd, block=True)
# import sql-dump
cmd = self.db_import_cmd.format(db="new_nextcloud",
pwd=pwd,
path=src_sql_path.as_posix())
cr = CommandRunner(cmd, block=True, shell=True)
# exit here if import failed, no changes done to live-db (nextcloud)
if cr.returncode != 0:
cr.log_output()
log.error("failed importing sql-dump into new database")
return False
log.info("success importing sql-dump into temp database")
try:
self.nc.set_maintenance_on()
except NextcloudError as e:
log.error(
"could not switch on maintainance mode, stopping restore")
log.error(exc_info=e)
return False
# drop databases (old_nextcloud)
cmd = self.db_cmd.format(pwd=pwd,
sql="DROP DATABASE IF EXISTS old_nextcloud")
cr = CommandRunner.retries(5, cmd, block=True)
# create new database (old_nextcloud) to move 'nextcloud' into
cmd = self.db_cmd.format(pwd=pwd, sql="CREATE DATABASE old_nextcloud")
cr = CommandRunner.retries(5, cmd, block=True)
# move current to old_nextcloud, make thus nextcloud will be empty
res = self.sql_move_all_tables(pwd, "nextcloud", "old_nextcloud")
if not res:
log.error(
"failed moving tables from 'nextcloud' to 'old_nextcloud'")
# move newly imported database into live database nextcloud
self.sql_move_all_tables(pwd, "new_nextcloud", "nextcloud")
if not res:
log.error(
"failed moving tables from 'new_nextcloud' to 'nextcloud'")
try:
self.nc.set_maintenance_off()
except NextcloudError as e:
log.error(
"could not switch off maintainance mode, stopping restore",
exc_info=e)
return False
log.info("completed sql-database import")
return True
def log_output(self):
log.info(f"#### log output for command: {self.cmd}")
for line in self.output:
log.info(line)
log.info(f"#### log output end")
def full_import(self, src_path):
if not isinstance(src_path, Path):
src_path = Path(src_path)
steps = [
("sql", lambda: self.import_sql(src_path)),
("config", lambda: self.import_config_dir(src_path)),
("nextbox", lambda: self.import_nextbox_dir(src_path)),
("data", lambda: self.import_dir("data", src_path)),
("apps", lambda: self.import_dir("apps", src_path)),
("letsencrypt", lambda: self.import_dir("letsencrypt", src_path)),
]
if not src_path.exists():
os.makedirs(src_path.as_posix())
self.check_backup(src_path)
failed = False
log.info("starting full import")
for step_key, step_func in steps:
log.debug(f"full import step: {step_key}")
# blocking step (sql)
ret = step_func()
# blocking call, eval directly and yield state
if step_key in ["sql", "nextbox", "config"]:
if ret == True:
log.debug(f"finished import step: {step_key}")
yield ("finished", (step_key, "import"), 100)
continue
else:
log.debug(f"failed import step: {step_key}")
failed = True
yield ("failed", (step_key, "import"), 100)
break
# non-blocking call(s) check_progress() and yield state
while True:
act, desc, percent = self.check_progress()
if act != "active":
if act == "failed":
log.debug(f"failed import step: {step_key}")
failed = True
else:
log.debug(f"finished import step: {step_key}")
yield (act, desc, 100)
break
time.sleep(1)
yield (act, desc, percent)
# final restore steps
if not failed:
# recreate apache config based on current (imported) config
conf_path = Path(self.dirs["nextbox"]) / self.nextbox_conf
new_cfg = yaml.safe_load(conf_path.open()).get("config", {})
has_https = new_cfg.get("https_port")
if has_https:
domain = new_cfg.get("domain")
certs = Certificates()
my_cert = certs.get_cert(domain)
if not my_cert:
log.error(
f"expected https/ssl, but could not find certificate: {domain}"
)
log.error("switching apache2 config to non-ssl")
certs.set_apache_config(ssl=False)
#### naaah this is evil:
new_cfg["https_port"] = None
dct = {"config": new_cfg}
yaml.safe_dump(dct, conf_path.open("w"))
else:
log.info(
f"found certificate for {domain} - activating ssl")
certs.write_apache_ssl_conf(my_cert["domains"][0],
my_cert["fullchain_path"],
my_cert["privkey_path"])
if not certs.set_apache_config(ssl=True):
log.error("failed enabling ssl config for apache")
else:
log.info("re-enabled ssl for imported configuration")
log.debug("wrote apache config according to restored data")
# restart daemon
log.info("finalized import - all seems good!")
log.info(".... restarting daemon")
services.restart("nextbox-daemon")
yield ("completed", ("all", "import"), 100)
def handle_config():
if request.method == "GET":
data = dict(cfg["config"])
try:
data["conf"] = Path(DDCLIENT_CONFIG_PATH).read_text("utf-8")
except FileNotFoundError:
data["conf"] = ""
return success(data=data)
# save dyndns related values to configuration
elif request.method == "POST":
run_jobs = []
for key in request.form:
val = request.form.get(key)
# special config-value 'conf' represents ddclient-config-contents
if key == "conf":
old_conf = Path(DDCLIENT_CONFIG_PATH).read_text("utf-8")
if old_conf != val:
log.info("writing ddclient config and restarting service")
Path(DDCLIENT_CONFIG_PATH).write_text(val, "utf-8")
elif len(val.strip()) == 0:
log.info("writing empty ddclient config")
Path(DDCLIENT_CONFIG_PATH).write_text(val, "utf-8")
services.stop("ddclient")
services.disable("ddclient")
if len(val.strip()) > 0:
services.enable("ddclient")
services.restart("ddclient")
run_jobs.append("DynDNSUpdate")
elif key in AVAIL_CONFIGS and val is not None:
# only allow valid DYNDNS_MODES
if key == "dns_mode" and val not in DYNDNS_MODES:
log.warning(
f"key: 'dns_mode' has invalid value: {val} - skipping")
continue
# start DynDNS update on "desec_done"
elif key == "dns_mode" and val == "desec_done":
run_jobs.append("DynDNSUpdate")
# start TrustedDomains update on new domain
elif "domain" in key:
run_jobs.append("TrustedDomains")
# deactivate proxy on request
elif key == "proxy_active" and val.lower() == "false":
proxy_tunnel = ProxyTunnel()
proxy_tunnel.stop()
# skip if 'val' is empty
elif val is None:
log.debug(f"skipping key: '{key}' -> no value provided")
continue
# convert to bool, ugly?
if val.lower() in ["true", "false"]:
val = val.lower() == "true"
# put key-value into cfg and save (yes, saving each value)
cfg["config"][key] = val
log.debug(f"saving key: '{key}' with value: '{val}'")
cfg.save()
# run jobs collected during configuration update
if len(run_jobs) > 0:
for job in run_jobs:
job_queue.put(job)
return success("DynDNS configuration saved")
def write_dedyn_credentials(self, domain, token):
with open(self.desec_credentials_path, "w") as fd:
fd.write(f"export DEDYN_TOKEN={token}\n")
fd.write(f"export DEDYN_NAME={domain}\n")
log.info(f"wrote {self.desec_credentials_path}")
