def renew_certs(self):
# here we try to acquire a new certificate using certbot
dns_mode = cfg.get("config", {}).get("dns_mode")
desec_token = cfg.get("config", {}).get("desec_token")
https_port = cfg.get("config", {}).get("https_port")
domain = cfg.get("config", {}).get("domain")
# desec based certbot command (dns-based verification)
if dns_mode == "desec_done":
self.write_dedyn_credentials(domain, desec_token)
cmd = self.desec_renew_cmd.format(config_dir=self.config_dir)
# others use reachability based verification
elif dns_mode in ["config_done", "static_done"]:
cmd = self.renew_cmd.format(config_dir=self.config_dir,
token=desec_token)
else:
log.error("trying to renew certificate w/o activated certificate")
return False
# run certbot renew command
cr = CommandRunner(cmd, block=True)
if cr.returncode == 0:
return True
# on fail log output
log.error("could not renew certificate(s)")
cr.log_output()
return False
def acquire_cert(self, domain, email):
# here we try to acquire a new certificate using certbot
dns_mode = cfg.get("config", {}).get("dns_mode")
desec_token = cfg.get("config", {}).get("desec_token")
# desec based certbot command (dns-based verification)
if dns_mode == "desec_done":
self.write_dedyn_credentials(domain, desec_token)
cmd = self.desec_acquire_cmd.format(email=email,
domain=domain,
config_dir=self.config_dir)
# others use reachability based verification
elif dns_mode in ["config_done", "static_done"]:
cmd = self.acquire_cmd.format(email=email,
domain=domain,
config_dir=self.config_dir,
token=desec_token)
else:
log.error("trying to acquire certificate w/o finished dns-config")
return False
# run certbot acquire command
cr = CommandRunner(cmd, block=True)
if cr.returncode == 0:
return True
# on fail log output
log.error("could not acquire certificate")
cr.log_output()
return False
def run_cmd(self, *args):
"""
Run `occ` command with given `args`
* raise `NextcloudError` on error
* return (merged stdin + stderr) output on success
"""
cr = CommandRunner(self.occ_cmd + args, block=True)
if cr.returncode != 0:
cr.log_output()
raise NextcloudError("failed to execute nextcloud:occ command")
return cr.output[:-2]
def get_logs():
log_dir = Path("/srv/logdump")
journal_logs = {
"nextbox-daemon": "journald.nextbox-daemon.log",
"nextbox-compose": "journald.nextbox-compose.log"
}
cmd_dump_journalctl = "journalctl --no-pager -n 1000 -xeu {daemon} > {log_dir}/{filename}"
var_logfiles = [
"/var/log/dpkg.log",
"/var/log/unattended-upgrades/unattended-upgrades.log",
"/var/log/unattended-upgrades/unattended-upgrades-dpkg.log",
"/var/log/unattended-upgrades/unattended-upgrades-shutdown.log",
"/var/log/kern.log", "/var/log/letsencrypt/letsencrypt.log",
LOG_FILENAME
]
logfiles = []
# cleanup logdump dir
shutil.rmtree(log_dir.as_posix())
os.makedirs(log_dir.as_posix())
for unit, fn in journal_logs.items():
cmd = cmd_dump_journalctl.format(daemon=unit,
filename=fn,
log_dir=log_dir)
CommandRunner(cmd, block=True, shell=True)
logfiles.append(log_dir / fn)
for path in var_logfiles:
if Path(path).exists():
shutil.copy(path, log_dir.as_posix())
logfiles.append(log_dir / Path(path).name)
hash_file = Path(log_dir) / "sha256.txt"
CommandRunner(f"sha256sum {log_dir.as_posix()}/* > {hash_file.as_posix()}",
shell=True,
block=True)
logfiles.append(hash_file)
zip_path = "/srv/nextbox-logs.zip"
with ZipFile(zip_path, "w") as fd:
for path in logfiles:
fd.write(path, path.name)
with open(zip_path, "rb") as fd:
return success(data={"zip": b64encode(fd.read())})
def rsync_dir(self, key, src_dir, tar_dir, block_main=False):
"""
rsync is used to copy files for export & import.
* a --dry-run is parsed 1st to determine the number of files (paths) to be created on the target side
* -av is used to get one line of output for each "created" file (path), which determines the percentage
"""
cmd = self.rsync_stats_cmd.format(src=src_dir, tar=tar_dir)
cr = CommandRunner(cmd, block=True)
stats = {"key": key}
parsing = {
"Number of files:": "all",
"Number of created files:": "create",
"Number of deleted files:": "delete",
"Number of regular files transferred:": "transfer",
"Total file size:": "size"
}
for line in cr.output:
for needle, tag in parsing.items():
if needle in line:
try:
num_tok = line.split(":")[1].strip().split(
" ")[0].strip()
stats[tag] = int(num_tok.replace(",", ""))
except ValueError:
log.warning(f"failed parsing: {tag} - line: {line}")
break
# all what we need is parsed
if len(stats) >= 6:
break
self.rsync_stats = stats
def parse(line, data_dct):
data_dct.setdefault("line", 0)
data_dct.setdefault("ratio", 0)
data_dct["num_files"] = stats
data_dct["line"] += 1
cnt = stats.get("create", 0)
if cnt == 0:
data_dct["ratio"] = 100
else:
data_dct["ratio"] = int(
round(min(1, max(0, data_dct["line"] / cnt)), 2) * 100)
cmd = self.rsync_base_cmd.format(src=src_dir, tar=tar_dir)
self.rsync_proc = CommandRunner(cmd, cb_parse=parse, block=block_main)
return False
def get_local_certs(self):
"""Get the local certificate paths"""
certs = []
cr = CommandRunner(self.list_cmd.format(config_dir=self.config_dir),
block=True)
cur_cert = {}
for line in cr.output:
if "Certificate Name:" in line:
cur_cert = {}
cur_cert["name"] = line.split(":")[1].strip()
elif "Domains:" in line:
cur_cert["domains"] = line.split(":")[1].strip().split(" ")
elif "Certificate Path:" in line:
cur_cert["fullchain_path"] = line.split(":")[1].strip() \
.replace(self.config_dir, self.inside_docker_dir)
elif "Private Key Path:" in line:
cur_cert["privkey_path"] = line.split(":")[1].strip() \
.replace(self.config_dir, self.inside_docker_dir)
certs.append(cur_cert)
elif "Expiry Date:" in line:
cur_cert["expiry"] = line.split(":", 1)[1].strip()
return certs
def export_sql(self, tar_path):
db_env_dct = self.get_env_data()
pwd = db_env_dct["MYSQL_ROOT_PASSWORD"]
if pwd is None:
log.error("cannot get sql password for export, aborting...")
return False
try:
self.nc.set_maintenance_on()
except NextcloudError as e:
log.error(
"could not switch on maintainance mode, stopping restore",
exc_info=e)
return False
tar_sql_path = Path(tar_path) / self.sql_dump_fn
cmd = self.db_export_cmd.format(pwd=pwd, path=tar_sql_path.as_posix())
cr = CommandRunner(cmd, block=True, shell=True)
try:
self.nc.set_maintenance_off()
except NextcloudError as e:
log.error(
"could not switch off maintainance mode, stopping restore",
exc_info=e)
return False
upd = {"size_sql": os.path.getsize(tar_sql_path.as_posix())}
self.update_meta(tar_path, upd)
return cr.returncode == 0
def sql_move_all_tables(self, root_pwd, src_db, tar_db):
# get all tables from source db
cmd = self.db_cmd.format(pwd=root_pwd,
sql=f"USE {src_db}; show tables")
cr = CommandRunner.retries(5, cmd, block=True)
if cr.returncode != 0:
log.error(f"failed determining all tables in {src_db}")
return False
# build mass-table-rename query and run it to move 'src_db.*' to 'tar_db.*'
tables = [f"{src_db}.{tbl.strip()} to {tar_db}.{tbl.strip()}" \
for tbl in cr.output if tbl.strip()]
query = "rename table " + ", ".join(tables)
cmd = self.db_cmd.format(pwd=root_pwd, sql=query)
cr = CommandRunner.retries(5, cmd, block=True)
return cr.returncode == 0
def delete_cert(self, domain):
certs = self.get_local_certs()
for cert in certs:
if domain in cert["domains"]:
cmd = self.delete_cmd.format(config_dir=self.config_dir,
domain=domain)
cr = CommandRunner(cmd, block=True)
return cr.returncode == 0
log.warning(f"{domain} not in certbot certificates for deletion found")
return False
def test_ddclient():
cr = CommandRunner([DDCLIENT_BIN, "-verbose", "-foreground", "-force"],
block=True)
cr.log_output()
for line in cr.output:
if "SUCCESS:" in line:
return success("DDClient test: OK")
if "Request was throttled" in line:
pat = "available in ([0-9]*) seconds"
try:
waitfor = int(re.search(pat, line).groups()[0]) + 5
except:
waitfor = 10
return error("DDClient test: Not OK",
data={
"reason": "throttled",
"waitfor": waitfor
})
return error("DDClient test: Not OK", data={"reason": "unknown"})
def get_stats(self, tar_dir=None, strict=True):
src_dir = self.backup_dir
# if 'tar_dir' is provided, 'base_dir' is used as 'root_dir'
if tar_dir is not None and not isinstance(tar_dir, Path):
tar_dir = Path(tar_dir)
stats = {}
cmd = self.rsync_stats_cmd if not tar_dir else self.rsync_transfer_stats_cmd
src_paths = self.get_component_paths(src_dir)
tar_paths = self.get_component_paths(tar_dir) if tar_dir else {}
for part in self.components:
src = src_paths[part]
fmt_dct = {"src": src.as_posix()}
# for sql
if not strict and not src.exists():
continue
if tar_dir:
# skip entries w/o path
if tar_paths[part] is None:
continue
tar = tar_paths[part].parent
fmt_dct["tar"] = tar.as_posix()
# for sql (target sql file won't exist)
if part == "sql":
continue
cr = CommandRunner(cmd.format(**fmt_dct), block=True)
raw_stats = []
found = False
for line in cr.output:
if not found and not line.startswith("Number"):
continue
found = True
raw_stats.append(line)
# parse and save stats
stats[part] = self._parse_rsync_stats(raw_stats,
no_transfer=tar_dir is None)
return stats
def exec(self, name, op):
"""Execute 'op(eration)' on service identified with 'name'"""
# check if operation (op) is allowed for this service (name)
if not self.check(name, op):
log.error(f"failed check at exec, service: {name} operation: {op}")
return {}
# get full service name (including arg if applicable)
service, _ = self.SERVICES_CTRL[name]
# run 'systemctl' command
cr = CommandRunner([SYSTEMCTL_BIN, op, service], block=True)
output = [x for x in cr.output if x]
return {
"service": name,
"operation": op,
"return-code": cr.returncode,
"output": output
}
def poweroff():
log.info("POWER-OFF - by /poweroff request")
cr = CommandRunner("poweroff")
if cr.returncode != 0:
return error("failed executing: 'poweroff'")
return success(data={})
def reboot():
log.info("REBOOTING NOW - by /reboot request")
cr = CommandRunner("reboot")
if cr.returncode != 0:
return error("failed executing: 'reboot'")
return success(data={})
def reload_apache(self):
self.ensure_apache_mods()
cr = CommandRunner(self.apache_restart, block=True)
return cr.returncode == 0
def import_sql(self, src_path):
db_env_dct = self.get_env_data()
pwd = db_env_dct["MYSQL_ROOT_PASSWORD"]
if pwd is None:
log.error("cannot get (root) sql password for import, aborting...")
return False
src_sql_path = Path(src_path) / self.sql_dump_fn
if not src_sql_path.exists():
log.error("sql-import data path not found, aborting...")
return False
# drop (new) database
cmd = self.db_cmd.format(pwd=pwd,
sql="DROP DATABASE IF EXISTS new_nextcloud")
cr = CommandRunner.retries(5, cmd, block=True)
# create new database
cmd = self.db_cmd.format(pwd=pwd, sql="CREATE DATABASE new_nextcloud")
cr = CommandRunner.retries(5, cmd, block=True)
# import sql-dump
cmd = self.db_import_cmd.format(db="new_nextcloud",
pwd=pwd,
path=src_sql_path.as_posix())
cr = CommandRunner(cmd, block=True, shell=True)
# exit here if import failed, no changes done to live-db (nextcloud)
if cr.returncode != 0:
cr.log_output()
log.error("failed importing sql-dump into new database")
return False
log.info("success importing sql-dump into temp database")
try:
self.nc.set_maintenance_on()
except NextcloudError as e:
log.error(
"could not switch on maintainance mode, stopping restore")
log.error(exc_info=e)
return False
# drop databases (old_nextcloud)
cmd = self.db_cmd.format(pwd=pwd,
sql="DROP DATABASE IF EXISTS old_nextcloud")
cr = CommandRunner.retries(5, cmd, block=True)
# create new database (old_nextcloud) to move 'nextcloud' into
cmd = self.db_cmd.format(pwd=pwd, sql="CREATE DATABASE old_nextcloud")
cr = CommandRunner.retries(5, cmd, block=True)
# move current to old_nextcloud, make thus nextcloud will be empty
res = self.sql_move_all_tables(pwd, "nextcloud", "old_nextcloud")
if not res:
log.error(
"failed moving tables from 'nextcloud' to 'old_nextcloud'")
# move newly imported database into live database nextcloud
self.sql_move_all_tables(pwd, "new_nextcloud", "nextcloud")
if not res:
log.error(
"failed moving tables from 'new_nextcloud' to 'nextcloud'")
try:
self.nc.set_maintenance_off()
except NextcloudError as e:
log.error(
"could not switch off maintainance mode, stopping restore",
exc_info=e)
return False
log.info("completed sql-database import")
return True
def clear_dns_caches(self):
CommandRunner([SYSTEMD_RESOLVE_BIN, "--flush-cache"], block=True)
CommandRunner([SYSTEMD_RESOLVE_BIN, "--reset-server-features"],
block=True)