def run(self): # behold the mighty worker-loop while True: pending_jobs = [] try: job = self.my_job_queue.get(timeout=1) if isinstance(job, str): # handle job w/o args, comes as `name: str` pending_jobs.append((job, None)) else: # already comes as `(name: str, args: tupel)` pending_jobs.append(job) except Empty: # returns: list[job: tuple[str, tuple]] pending_jobs = self.job_mgr.get_recurring_jobs() # nothing to do => power nap if len(pending_jobs) == 0: sleep(0.5) continue for job_name, job_args in pending_jobs: # special job "exit" will stop the worker-queue if job_name == "exit": log.info("exiting (background worker) by request") return # finally handle (dispatch): `job_name` + `job_args` self.job_mgr.handle_job(job_name, job_args)
def import_config_dir(self, src_path): # regulary copy over all files self.import_dir("config", src_path, block=True) log.info("copied configs done - updating db-password") # afterwards make sure the password for sql is correct inside config.php cfg_path = Path(self.dirs["config"]) / "config.php" cfg_content = cfg_path.read_text() # get pass from docker.env db_env_dct = self.get_env_data() dbpass = db_env_dct["MYSQL_PASSWORD"] # read config, regex dbpassword, replace with new pass (from docker.env) pat = re.compile(r"'dbpassword'[^=]*=>[^']*'([^']*)'[^,]*,") res = pat.search(cfg_content) old_pass = res.group() new_pass = f"'dbpassword' => '{dbpass}'," with cfg_path.open("w") as fd: fd.write(cfg_content.replace(old_pass, new_pass)) log.info("updated config.php with 'nextcloud' db password") return True
def register_job(self, job): """register job by `job.name` and construct their instances""" if job.name in self.jobs: log.warning(f"overwriting job (during register) with name: {job.name}") self.jobs[job.name] = job() log.info(f"registered job: {job.name}")
def export_dir(self, key, tar_dir): self.activity_desc = (key, "export") self.rsync_dir(key, self.dirs[key], tar_dir) log.info(f"starting {key} export") if self.rsync_stats: self.update_meta(tar_dir, {"size_" + key: self.rsync_stats.get("size", 0)})
def import_dir(self, key, src_dir, block=False): self.activity_desc = (key, "import") src_dir = Path(src_dir) key_path = Path(self.dirs[key]) src_path = src_dir / key_path.name tar_path = key_path.parent self.rsync_dir("data", src_path.as_posix(), tar_path.as_posix(), block_main=block) log.info(f"starting {key} import")
def _run(self, cfg, board, kwargs): # only enable, if nextcloud is installed if not self.nc.is_installed: log.debug("cannot enable nextbox-app - uninitialized nextcloud") return # try to enable try: if self.nc.enable_nextbox_app(): self.interval = 3600 log.info("enabled nextcloud nextbox-app") except NextcloudError: pass
def restore_start(): src_path = request.form.get("src_path") if not backup_restore.check_backup(src_path): msg = "Invalid backup, cannot restore" log.error(msg) return error(msg) log.info(f"Initiating restore from: {src_path}") job_kwargs = {"tar_path": src_path, "mode": "restore"} job_queue.put(("BackupRestore", job_kwargs)) return success("restore started")
def setup(self, token, subdomain, scheme): if scheme not in ["http", "https"]: raise ProxySetupError( f"provided scheme: {scheme} not 'http' or 'https'") local_port = 80 if scheme == "http" else 443 remote_port = self.register_at_server(token, subdomain, scheme) self.create_config(token, remote_port, local_port) self.start() log.info("setup proxy:") log.info(f"port: {local_port}, remote_port: {remote_port} @ {scheme}") return remote_port
def ensure_apache_mods(self): avail = Path(self.apache_mods_available) enabled = Path(self.apache_mods_enabled) for mod in self.apache_mods: conf_avail_path = (avail / (mod + ".conf")) load_avail_path = (avail / (mod + ".load")) conf_enabled_path = (enabled / (mod + ".conf")) load_enabled_path = (enabled / (mod + ".load")) if conf_avail_path.exists() and not conf_enabled_path.exists(): conf_enabled_path.symlink_to("../mods-available/" + conf_avail_path.name) log.info(f"enabled: {conf_avail_path.name}") if load_avail_path.exists() and not load_enabled_path.exists(): load_enabled_path.symlink_to("../mods-available/" + load_avail_path.name) log.info(f"enabled: {load_avail_path.name}")
def _run(self, cfg, board, kwargs): token = cfg["config"]["desec_token"] dns_mode = cfg["config"]["dns_mode"] domain = cfg["config"]["domain"] if dns_mode not in ["desec_done", "off"]: if not services.is_active("ddclient"): services.restart("ddclient") services.enable("ddclient") return if dns_mode == "off": if services.is_active("ddclient"): services.stop("ddclient") services.disable("ddclient") return if dns_mode == "desec_done": if services.is_active("ddclient"): services.stop("ddclient") services.disable("ddclient") dns = DNSManager() ipv4 = dns.get_ipv4() ipv6 = dns.get_ipv6() headers = {"Authorization": f"Token {token}"} params = {"hostname": domain} if ipv4: params["myipv4"] = ipv4 if ipv6: params["myipv6"] = ipv6 res = requests.get("https://update.dedyn.io", params=params, headers=headers) if res.ok: log.info(f"updated deSEC IPv4 ({ipv4}) and IPv6 ({ipv6}) address for '{domain}'") else: log.warning(f"failed updating IPs ({ipv4} & {ipv6}) for domain: '{domain}'") log.debug(f"result: {res.text} status_code: {res.status_code} url: {res.url}")
def _run(self, cfg, board, kwargs): self.interval = None # if the custom-dns-config is active, ensure ddclient is up if cfg["config"]["dns_mode"] == "config_done" and not services.is_active("ddclient"): services.restart("ddclient") # ensure that neither updater nor factory-reset are masked (paranoia!) services.unmask("nextbox-updater") services.unmask("nextbox-factory-reset") # log a welcome + version log.info(f"Hello World - I am NextBox - call me: v{nextbox_version()} - let'sa gooo") # update/upgrade now shield.set_led_state("updating") log.info("running 'apt-get update'") cache = apt.cache.Cache() cache.update() cache.open() # which debian package pkg = cfg["config"]["debian_package"] try: pkg_obj = cache[pkg] except KeyError: log.error(f"self-update failed: designated package: {pkg} not found!") log.error("falling back to 'nextbox' - retrying upgrade...") pkg = "nextbox" try: pkg_obj = cache[pkg] except KeyError: log.error("CRITICAL: failed to find 'nextbox' in apt-cache") # we should never ever end here, this means that the nextbox # debian package is not available... # nextbox debian (ppa) repository not available ???!! return # install package (i.e., other nextbox package is already installed) # will trigger for e.g., 'nextbox' to 'nextbox-testing' switching if not pkg_obj.is_installed: log.info(f"installing debian package: {pkg} (start service: nextbox-updater)") services.start("nextbox-updater") elif pkg_obj.is_upgradable: log.info(f"upgrading debian package: {pkg} (start service: nextbox-updater)") services.start("nextbox-updater") else: log.debug(f"no need to upgrade or install debian package: {pkg}")
def backup_start(): tar_path = request.form.get("tar_path") found = False for dev in partitions.backup_devices: if tar_path.startswith(dev["path"]): found = dev break if not found: msg = "Invalid backup location provided" log.error(msg) return error(msg) log.info( f"Initiating backup onto: {dev['friendly_name']} @ {dev['path']} with target: {tar_path}" ) job_kwargs = {"tar_path": tar_path, "mode": "backup"} job_queue.put(("BackupRestore", job_kwargs)) return success("backup started")
def ssh_set(): auth_p_dir = Path("/home/nextuser/.ssh") if not auth_p_dir.exists(): os.makedirs(auth_p_dir.as_posix()) auth_p = auth_p_dir / "authorized_keys" if request.method == "GET": ip = local_ip() if not auth_p.exists(): return success(data={"pubkey": "", "local_ip": ip}) with auth_p.open() as fd: return success(data={"pubkey": fd.read(), "local_ip": ip}) elif request.method == "POST": pubkey = request.form.get("pubkey") with auth_p.open("w") as fd: fd.write(pubkey.strip() + "\n") log.info(f"setting ssh pub key: {pubkey}") return success()
def test_resolve4(): ip_type = request.path.split("/")[-1] domain = cfg["config"]["domain"] resolve_ip = None ext_ip = None if not domain: return error("no domain is set") dns = DNSManager() dns.clear_dns_caches() if ip_type == "ipv4": resolve_ip = dns.resolve_ipv4(domain) ext_ip = dns.get_ipv4() else: resolve_ip = dns.resolve_ipv6(domain) ext_ip = dns.get_ipv6() log.info( f"resolving '{domain}' to IP: {resolve_ip}, external IP: {ext_ip}") data = {"ip": ext_ip, "resolve_ip": resolve_ip, "domain": domain} # if not both "resolve" and "getip" are successful, we have failed if resolve_ip is None or ext_ip is None: log.error(f"failed resolving and/or getting external {ip_type}") return error("Resolve test: Not OK", data=data) # resolving to wrong ip if resolve_ip != ext_ip: log.warning(f"Resolved {ip_type} does not match external {ip_type}") log.warning("This might indicate a bad DynDNS configuration") return error("Resolve test: Not OK", data=data) # all good! return success("Resolve test: OK", data=data)
def signal_handler(sig, frame): global job_queue, worker log.info("signal-handler, sending 'exit' as job for graceful termination") job_queue.put("exit") worker.join() log.info("joined background-worker - exiting now...") log.info("^" * 60) sys.exit(1)
def reboot(): log.info("REBOOTING NOW - by /reboot request") cr = CommandRunner("reboot") if cr.returncode != 0: return error("failed executing: 'reboot'") return success(data={})
def _run(self, cfg, board, kwargs): board.set("tls", { "what": "enable", "state": "running", }) certs = Certificates() domain = cfg.get("config", {}).get("domain") email = cfg.get("config", {}).get("email") if not domain or not email: log.error(f"cannot enable tls, domain: '{domain}' email: '{email}'") board.set("tls", { "what": "domain-or-email", "state": "fail", }) return False my_cert = certs.get_cert(domain) if not my_cert: log.warning(f"could not get local certificate for: {domain}, acquiring...") if not certs.acquire_cert(domain, email): msg = f"Failed to acquire {domain} with {email}" log.error(msg) board.set("tls", { "what": "acquire", "state": "fail", }) return False log.info(f"acquired certificate for: {domain} with {email}") my_cert = certs.get_cert(domain) certs.write_apache_ssl_conf( my_cert["domains"][0], my_cert["fullchain_path"], my_cert["privkey_path"] ) if not certs.set_apache_config(ssl=True): board.set("tls", { "what": "apache-config", "state": "fail", }) log.error("failed enabling ssl configuration for apache") return False log.info(f"activated https for apache using: {domain}") cfg["config"]["https_port"] = 443 cfg["config"]["email"] = email cfg.save() board.set("tls", { "what": "enable", "state": "success", }) # we need to "re-wire" the proxy to port 443 on activated TLS add_msg = "" if cfg["config"]["proxy_active"]: subdomain = cfg["config"]["proxy_domain"].split(".")[0] scheme = "https" token = cfg["config"]["nk_token"] proxy_tunnel = ProxyTunnel() try: proxy_port = proxy_tunnel.setup(token, subdomain, scheme) except ProxySetupError as e: log.error("register at proxy-server error", exc_info=e) except Exception as e: log.error("unexpected error during proxy setup", exc_info=e) cfg["config"]["proxy_port"] = proxy_port cfg.save()
def poweroff(): log.info("POWER-OFF - by /poweroff request") cr = CommandRunner("poweroff") if cr.returncode != 0: return error("failed executing: 'poweroff'") return success(data={})
def import_sql(self, src_path): db_env_dct = self.get_env_data() pwd = db_env_dct["MYSQL_ROOT_PASSWORD"] if pwd is None: log.error("cannot get (root) sql password for import, aborting...") return False src_sql_path = Path(src_path) / self.sql_dump_fn if not src_sql_path.exists(): log.error("sql-import data path not found, aborting...") return False # drop (new) database cmd = self.db_cmd.format(pwd=pwd, sql="DROP DATABASE IF EXISTS new_nextcloud") cr = CommandRunner.retries(5, cmd, block=True) # create new database cmd = self.db_cmd.format(pwd=pwd, sql="CREATE DATABASE new_nextcloud") cr = CommandRunner.retries(5, cmd, block=True) # import sql-dump cmd = self.db_import_cmd.format(db="new_nextcloud", pwd=pwd, path=src_sql_path.as_posix()) cr = CommandRunner(cmd, block=True, shell=True) # exit here if import failed, no changes done to live-db (nextcloud) if cr.returncode != 0: cr.log_output() log.error("failed importing sql-dump into new database") return False log.info("success importing sql-dump into temp database") try: self.nc.set_maintenance_on() except NextcloudError as e: log.error( "could not switch on maintainance mode, stopping restore") log.error(exc_info=e) return False # drop databases (old_nextcloud) cmd = self.db_cmd.format(pwd=pwd, sql="DROP DATABASE IF EXISTS old_nextcloud") cr = CommandRunner.retries(5, cmd, block=True) # create new database (old_nextcloud) to move 'nextcloud' into cmd = self.db_cmd.format(pwd=pwd, sql="CREATE DATABASE old_nextcloud") cr = CommandRunner.retries(5, cmd, block=True) # move current to old_nextcloud, make thus nextcloud will be empty res = self.sql_move_all_tables(pwd, "nextcloud", "old_nextcloud") if not res: log.error( "failed moving tables from 'nextcloud' to 'old_nextcloud'") # move newly imported database into live database nextcloud self.sql_move_all_tables(pwd, "new_nextcloud", "nextcloud") if not res: log.error( "failed moving tables from 'new_nextcloud' to 'nextcloud'") try: self.nc.set_maintenance_off() except NextcloudError as e: log.error( "could not switch off maintainance mode, stopping restore", exc_info=e) return False log.info("completed sql-database import") return True
def log_output(self): log.info(f"#### log output for command: {self.cmd}") for line in self.output: log.info(line) log.info(f"#### log output end")
def full_import(self, src_path): if not isinstance(src_path, Path): src_path = Path(src_path) steps = [ ("sql", lambda: self.import_sql(src_path)), ("config", lambda: self.import_config_dir(src_path)), ("nextbox", lambda: self.import_nextbox_dir(src_path)), ("data", lambda: self.import_dir("data", src_path)), ("apps", lambda: self.import_dir("apps", src_path)), ("letsencrypt", lambda: self.import_dir("letsencrypt", src_path)), ] if not src_path.exists(): os.makedirs(src_path.as_posix()) self.check_backup(src_path) failed = False log.info("starting full import") for step_key, step_func in steps: log.debug(f"full import step: {step_key}") # blocking step (sql) ret = step_func() # blocking call, eval directly and yield state if step_key in ["sql", "nextbox", "config"]: if ret == True: log.debug(f"finished import step: {step_key}") yield ("finished", (step_key, "import"), 100) continue else: log.debug(f"failed import step: {step_key}") failed = True yield ("failed", (step_key, "import"), 100) break # non-blocking call(s) check_progress() and yield state while True: act, desc, percent = self.check_progress() if act != "active": if act == "failed": log.debug(f"failed import step: {step_key}") failed = True else: log.debug(f"finished import step: {step_key}") yield (act, desc, 100) break time.sleep(1) yield (act, desc, percent) # final restore steps if not failed: # recreate apache config based on current (imported) config conf_path = Path(self.dirs["nextbox"]) / self.nextbox_conf new_cfg = yaml.safe_load(conf_path.open()).get("config", {}) has_https = new_cfg.get("https_port") if has_https: domain = new_cfg.get("domain") certs = Certificates() my_cert = certs.get_cert(domain) if not my_cert: log.error( f"expected https/ssl, but could not find certificate: {domain}" ) log.error("switching apache2 config to non-ssl") certs.set_apache_config(ssl=False) #### naaah this is evil: new_cfg["https_port"] = None dct = {"config": new_cfg} yaml.safe_dump(dct, conf_path.open("w")) else: log.info( f"found certificate for {domain} - activating ssl") certs.write_apache_ssl_conf(my_cert["domains"][0], my_cert["fullchain_path"], my_cert["privkey_path"]) if not certs.set_apache_config(ssl=True): log.error("failed enabling ssl config for apache") else: log.info("re-enabled ssl for imported configuration") log.debug("wrote apache config according to restored data") # restart daemon log.info("finalized import - all seems good!") log.info(".... restarting daemon") services.restart("nextbox-daemon") yield ("completed", ("all", "import"), 100)
def handle_config(): if request.method == "GET": data = dict(cfg["config"]) try: data["conf"] = Path(DDCLIENT_CONFIG_PATH).read_text("utf-8") except FileNotFoundError: data["conf"] = "" return success(data=data) # save dyndns related values to configuration elif request.method == "POST": run_jobs = [] for key in request.form: val = request.form.get(key) # special config-value 'conf' represents ddclient-config-contents if key == "conf": old_conf = Path(DDCLIENT_CONFIG_PATH).read_text("utf-8") if old_conf != val: log.info("writing ddclient config and restarting service") Path(DDCLIENT_CONFIG_PATH).write_text(val, "utf-8") elif len(val.strip()) == 0: log.info("writing empty ddclient config") Path(DDCLIENT_CONFIG_PATH).write_text(val, "utf-8") services.stop("ddclient") services.disable("ddclient") if len(val.strip()) > 0: services.enable("ddclient") services.restart("ddclient") run_jobs.append("DynDNSUpdate") elif key in AVAIL_CONFIGS and val is not None: # only allow valid DYNDNS_MODES if key == "dns_mode" and val not in DYNDNS_MODES: log.warning( f"key: 'dns_mode' has invalid value: {val} - skipping") continue # start DynDNS update on "desec_done" elif key == "dns_mode" and val == "desec_done": run_jobs.append("DynDNSUpdate") # start TrustedDomains update on new domain elif "domain" in key: run_jobs.append("TrustedDomains") # deactivate proxy on request elif key == "proxy_active" and val.lower() == "false": proxy_tunnel = ProxyTunnel() proxy_tunnel.stop() # skip if 'val' is empty elif val is None: log.debug(f"skipping key: '{key}' -> no value provided") continue # convert to bool, ugly? if val.lower() in ["true", "false"]: val = val.lower() == "true" # put key-value into cfg and save (yes, saving each value) cfg["config"][key] = val log.debug(f"saving key: '{key}' with value: '{val}'") cfg.save() # run jobs collected during configuration update if len(run_jobs) > 0: for job in run_jobs: job_queue.put(job) return success("DynDNS configuration saved")
def write_dedyn_credentials(self, domain, token): with open(self.desec_credentials_path, "w") as fd: fd.write(f"export DEDYN_TOKEN={token}\n") fd.write(f"export DEDYN_NAME={domain}\n") log.info(f"wrote {self.desec_credentials_path}")