def check_updates():
"""Check for updates from arkOS repo server."""
updates = []
gpg = gnupg.GPG()
server = config.get("general", "repo_server")
current = config.get("updates", "current_update")
# Fetch updates from registry server
api_url = "https://{0}/api/v1/updates/{1}"
data = api(api_url.format(server, str(current)), crit=True)
for x in data["updates"]:
ustr, u = str(x["tasks"]), json.loads(x["tasks"])
# Get the update signature and test it
sig_url = "https://{0}/api/v1/signatures/{1}"
sig = api(sig_url.format(server, x["id"]), returns="raw", crit=True)
with open("/tmp/{0}.sig".format(x["id"]), "w") as f:
f.write(sig)
v = gpg.verify_data("/tmp/{0}.sig".format(x["id"]), ustr)
if v.trust_level is None:
err_str = "Update {0} signature verification failed"
logger.error("Updates", err_str.format(x["id"]))
break
else:
data = {
"id": x["id"],
"name": x["name"],
"date": x["date"],
"info": x["info"],
"tasks": u
}
updates.append(data)
storage.updates = {x.id: x for x in updates}
return updates
def install(*mods, **kwargs):
"""
Install a set of NPM packages.
Include ``as_global`` in kwargs to install package as global.
:param *mods: Packages to install
:param **kwargs: Extra keyword arguments to pass to NPM
"""
as_global = "-g " if kwargs.get("as_global") else ""
mods = " ".join(x for x in mods)
opt_str = ""
if kwargs.get("opts"):
opt_str += " --"
for k, v in kwargs.get("opts", {}):
opt_str += " --".join(k + v if v[0] == '=' else k + " " + v)
cwd = os.getcwd()
if "install_path" in kwargs:
os.chdir(kwargs["install_path"])
s = shell("npm install {0}{1}{2}".format(
as_global,
mods,
))
os.chdir(cwd)
if s["code"] != 0:
errmsg = "NPM install of {0} failed.".format(mods)
logmsg = "NPM install failure details:\n{0}"
logger.error("NodeJS", logmsg.format(s["stderr"].decode()))
raise errors.OperationFailedError(errmsg)
def post_install(self, extra_vars, dbpasswd=""):
secret_key = random_string()
dbengine = 'mysql' \
if self.app.selected_dbengine == 'db-mariadb' \
else 'sqlite'
# Write a standard Wallabag config file
config_file = os.path.join(self.path, 'app/config/parameters.yml')
with open(config_file + ".dist", 'r') as f:
ic = f.readlines()
with open(config_file, 'w') as f:
for l in ic:
if "database_driver: " in l:
pdo = "pdo_mysql" if dbengine == "mysql" else "pdo_sqlite"
l = " database_driver: {0}\n".format(pdo)
elif "database_path: " in l and dbengine == 'sqlite':
l = " database_path: {0}\n".format(self.db.path)
elif "database_name: " in l and dbengine == 'mysql':
l = " database_name: {0}\n".format(self.db.id)
elif "database_user: "******" database_user: {0}\n".format(self.db.id)
elif "database_password: "******"{0}"\n'.format(dbpasswd)
elif "secret: " in l:
l = " secret: {0}\n".format(secret_key)
f.write(l)
# Make sure that the correct PHP settings are enabled
php.enable_mod('sqlite3', 'bcmath',
'pdo_mysql' if dbengine == 'mysql' else 'pdo_sqlite',
'zip', 'tidy')
php.open_basedir('add', '/usr/bin/php')
uid, gid = users.get_system("http").uid, groups.get_system("http").gid
# Set up the database then delete the install folder
if dbengine == 'sqlite3':
php.open_basedir('add', '/var/lib/sqlite3')
cwd = os.getcwd()
os.chdir(self.path)
s = shell("php bin/console wallabag:install --env=prod -n")
if s["code"] != 0:
logger.error("Websites", s["stderr"].decode())
raise errors.OperationFailedError(
"Failed to populate database. See logs for more info")
os.chdir(cwd)
if dbengine == 'sqlite3':
os.chown("/var/lib/sqlite3/{0}.db".format(self.db.id), -1, gid)
os.chmod("/var/lib/sqlite3/{0}.db".format(self.db.id), 0o660)
# Finally, make sure that permissions are set so that Wallabag
# can make adjustments and save plugins when need be.
for r, d, f in os.walk(self.path):
for x in d:
os.chown(os.path.join(r, x), uid, gid)
for x in f:
os.chown(os.path.join(r, x), uid, gid)
def post(self):
data = request.get_json()["database"]
manager = databases.get_managers(data["database_type"])
try:
db = manager.add_db(data["id"])
except errors.InvalidConfigError as e:
logger.error("Databases", str(e))
return jsonify(errors={"msg": str(e)}), 422
return jsonify(database=db.serialized)
def install_from_package(path, stat="production", opts={}):
# Installs a set of NPM package dependencies from an NPM package.json.
cwd = os.getcwd()
os.chdir(path)
s = shell("npm install %s%s" % (" --"+stat if stat else "", " --"+" --".join(x+"="+opts[x] for x in opts) if opts else ""))
os.chdir(cwd)
if s["code"] != 0:
logger.error("NPM install of %s failed; log output follows:\n%s"%(path,s["stderr"]))
raise Exception("NPM install failed, check logs for info")
def delete(self, id):
mount = sharers.get_mounts(id)
if not id or not mount:
abort(404)
try:
mount.remove()
except errors.InvalidConfigError as e:
logger.error("Sharers", str(e))
return jsonify(errors={"msg": str(e)}), 422
return Response(status=204)
def update_gem(*gems, **kwargs):
# Updates a set of presently-installed Ruby gems.
verify_path()
gemlist = shell("gem list")["stdout"].split("\n")
for x in gems:
if not any(x==s for s in gemlist) or force:
d = shell("gem update -N --no-user-install %s" % x)
if d["code"] != 0:
logger.error("Gem update '%s' failed: %s"%(x,str(d["stderr"])))
raise Exception("Gem update '%s' failed. See logs for more info"%x)
def pre_backup(self):
"""Reimplement."""
s = shell("slapcat -n 1")
if s["code"] != 0:
emsg = ("Could not backup LDAP database. "
"Please check logs for errors.")
logger.error("Backup", s["stderr"].decode())
raise errors.OperationFailedError(emsg)
with open("/tmp/ldap.ldif", "wb") as f:
f.write(s["stdout"])
def install_gem(*gems, **kwargs):
# Installs a set of Ruby gems to the system.
verify_path()
gemlist = shell("gem list")["stdout"].split("\n")
for x in gems:
if not any(x==s for s in gemlist) or force:
d = shell("gem install -N --no-user-install %s" % x)
if d["code"] != 0:
logger.error("Gem install '%s' failed: %s"%(x,str(d["stderr"])))
raise Exception("Gem install '%s' failed. See logs for more info"%x)
def delete(self, id):
db = databases.get(id)
if not id or not db:
abort(404)
try:
db.remove()
except errors.InvalidConfigError as e:
logger.error("Databases", str(e))
return jsonify(errors={"msg": str(e)}), 422
return Response(status=204)
def install(*mods, **kwargs):
# Installs a set of NPM packages.
as_global = kwargs.get("as_global", False)
cwd = os.getcwd()
if "install_path" in kwargs:
os.chdir(kwargs["install_path"])
s = shell("npm install %s%s%s" % ("-g " if as_global else "", " ".join(x for x in mods), (" --"+" --".join(k+v if v[0]=='=' else k+" "+v for k,v in kwargs["opts"].items()) if kwargs.has_key("opts") else "")))
os.chdir(cwd)
if s["code"] != 0:
logger.error("NPM install of %s failed; log output follows:\n%s"%(" ".join(x for x in mods),s["stderr"]))
raise Exception("NPM install failed, check logs for info")
def request(domain):
"""Request a free ACME certificate from Let's Encrypt."""
if not domain:
logger.error(
"ctl:info:generate", "Choose a fully-qualified domain name of the "
"certificate. Must match a domain present on the system")
domain = click.prompt("Domain name")
try:
certificates.request_acme_certificate(domain)
except Exception as e:
raise CLIException(str(e))
def post(self):
data = request.get_json()["mount"]
manager = sharers.get_sharers(data["share_type"])
try:
mount = manager.add_mount(data["path"], data["network_path"],
data.get("username"),
data.get("password"), data["read_only"])
except errors.InvalidConfigError as e:
logger.error("Sharers", str(e))
return jsonify(errors={"msg": str(e)}), 422
return jsonify(mount=mount.serialized)
def start(name):
"""Start a service"""
try:
service = services.get(name)
service.start()
if service.state == "running":
logger.success('ctl:svc:start', 'Started {0}'.format(name))
else:
logger.error('ctl:svc:start', 'Failed to start {0}'.format(name))
except Exception as e:
raise CLIException(str(e))
def remove(*mods):
"""
Remove a set of Python packages from the system.
:param *mods: packages to remove
"""
s = shell("pip uninstall {0}".format(mods))
if s["code"] != 0:
errmsg = "PyPI uninstall of {0} failed.".format(mods)
logmsg = "PyPI uninstall failure details:\n{0}"
logger.error("Python", logmsg.format(s["stderr"].decode()))
raise errors.OperationFailedError(errmsg)
def post(self):
data = request.get_json()["share"]
if not data.get("share_type"):
abort(422)
manager = sharers.get_sharers(data["share_type"])
try:
share = manager.add_share(data["id"], data["path"],
data.get("comment", ""),
data["valid_users"], data["read_only"])
except errors.InvalidConfigError as e:
logger.error("Sharers", str(e))
return jsonify(errors={"msg": str(e)}), 422
return jsonify(share=share.serialized)
def remove(*mods):
"""
Remove a set of NPM packages.
:param *mods: Packages to remove
"""
mods = " ".join(x for x in mods)
s = shell("npm uninstall {0}".format(mods), stderr=True)
if s["code"] != 0:
errmsg = "NPM removal of {0} failed.".format(mods)
logmsg = "NPM remove failure details:\n{0}"
logger.error("NodeJS", logmsg.format(s["stderr"].decode()))
raise errors.OperationFailedError(errmsg)
def perform_action(id, action):
w = websites.get(id)
if not w:
abort(404)
if not hasattr(w, action):
abort(422)
actionfunc = getattr(w, action)
try:
actionfunc()
except Exception as e:
logger.error("Websites", str(e))
return jsonify(errors={"msg": str(e)}), 500
finally:
return Response(status=200)
def update_site(self, pkg, ver):
cwd = os.getcwd()
os.chdir(self.path)
s = shell("bin/gpm selfupgrade -f")
if s["code"] != 0:
logger.error(
"Webs", "Grav failed to run selfupgrade. Error: {0}".format(
s["stderr"].decode()))
s = shell("bin/gpm update -f")
if s["code"] != 0:
logger.error(
"Webs", "Grav failed to run plugin update. Error: {0}".format(
s["stderr"].decode()))
os.chdir(cwd)
def generate_dh_params(path, size=2048):
"""
Create and save Diffie-Hellman parameters.
:param str path: File path to save to
:param int size: Key size
"""
s = shell("openssl dhparam {0} -out {1}".format(size, path))
if s["code"] != 0:
emsg = "Failed to generate Diffie-Hellman parameters"
logger.error("Certificates", s["stderr"].decode())
raise errors.OperationFailedError(emsg)
os.chown(path, -1, gid)
os.chmod(path, 0o750)
def install(pkg, version=None, py2=False):
"""
Install a set of Python packages from PyPI.
:param str pkg: package to install
:param str version: If present, install this specific version
:param bool py2: If True, install for Python 2.x instead
"""
if version:
pkg = pkg + "==" + version
s = shell("pip{0} install {1}".format("2" if py2 else "", pkg))
if s["code"] != 0:
errmsg = "PyPI install of {0} failed.".format(pkg)
logmsg = "PyPI install failure details:\n{0}"
logger.error("Python", logmsg.format(s["stderr"].decode()))
raise errors.OperationFailedError(errmsg)
def install_from_package(path, stat="production", opts={}):
"""
Install a set of NPM package dependencies from an NPM package.json.
:param str path: path to directory with package.json
:param str stat: Install to "production"?
"""
stat = (" --" + stat) if stat else ""
opts = (" --" + " --".join(x + "=" + y for x, y in opts)) if opts else ""
cwd = os.getcwd()
os.chdir(path)
s = shell("npm install {0}{1}".format(stat, opts))
os.chdir(cwd)
if s["code"] != 0:
errmsg = "NPM install of {0} failed.".format(path)
logmsg = "NPM install failure details:\n{0}"
logger.error("NodeJS", logmsg.format(s["stderr"].decode()))
raise errors.OperationFailedError(errmsg)
def install_composer():
"""Install Composer to the system."""
cwd = os.getcwd()
os.chdir("/root")
os.environ["COMPOSER_HOME"] = "/root"
enable_mod("phar")
open_basedir("add", "/root")
r = requests.get("https://getcomposer.org/installer")
s = shell("php", stdin=r.text)
os.chdir(cwd)
if s["code"] != 0:
errmsg = "Composer download/config failed."
logmsg = "Composer install failure details:\n{0}"
logger.error("PHP", logmsg.format(s["stderr"].decode()))
raise errors.OperationFailedError(errmsg)
os.rename("/root/composer.phar", "/usr/local/bin/composer")
os.chmod("/usr/local/bin/composer", 0o755)
open_basedir("add", "/usr/local/bin")
def install(gem, version=None, update=False):
"""
Install a Ruby gem to the system.
:param str gem: Gem name
:param str version: If present, install this specific version
:param bool update: If true, force an update
"""
verify_path()
if version:
gem = gem + ":" + version
s = shell("gem {0} -N --no-user-install {1}".format(
"update" if update else "install", gem))
if s["code"] != 0:
errmsg = "Gem install of {0} failed.".format(gem)
logmsg = "Gem install failure details:\n{0}"
logger.error("Ruby", logmsg.format(s["stderr"].decode()))
raise errors.OperationFailedError(errmsg)
def post_restore(self):
"""Reimplement."""
if not os.path.exists("/tmp/ldap.ldif"):
emsg = ("Could not backup LDAP database. "
"Please check logs for errors.")
logger.error("Backup", "/tmp/ldap.ldif not found")
raise errors.OperationFailedError(emsg)
with open("/tmp/ldap.ldif", "r") as f:
ldif = f.read()
s = shell('ldapadd -D "cn=admin,dc=arkos-servers,dc=org" -w {0}'
.format(secrets.get("ldap")),
stdin=ldif)
if os.path.exists("/tmp/ldap.ldif"):
os.unlink("/tmp/ldap.ldif")
if s["code"] != 0:
emsg = ("Could not restore LDAP database. "
"Please check logs for errors.")
logger.error("Backup", s["stderr"].decode())
raise errors.OperationFailedError(emsg)
def open_upnp(port):
"""
Open and forward a port with the local uPnP IGD.
:param tuple port: Port protocol and number
"""
upnpc = _upnp_igd_connect()
if not upnpc:
return
if upnpc.getspecificportmapping(int(port[1]), port[0].upper()):
try:
upnpc.deleteportmapping(int(port[1]), port[0].upper())
except:
pass
try:
pf = 'arkOS Port Forwarding: {0}'
upnpc.addportmapping(int(port[1]), port[0].upper(), upnpc.lanaddr,
int(port[1]), pf.format(port[1]), '')
except Exception as e:
msg = "Failed to register {0} with uPnP IGD: {1}"
logger.error("TrSv", msg.format(port, str(e)))
def open_all_upnp(ports):
"""
Open and forward multiple ports with the local uPnP IGD.
:param list ports: List of port objects to open
"""
upnpc = _upnp_igd_connect()
if not upnpc:
return
for port in [x for x in ports]:
if upnpc.getspecificportmapping(port[1], port[0].upper()):
try:
upnpc.deleteportmapping(port[1], port[0].upper())
except:
pass
try:
pf = 'arkOS Port Forwarding: {0}'
upnpc.addportmapping(port[1], port[0].upper(), upnpc.lanaddr,
port[1], pf.format(port[1]), '')
except Exception as e:
msg = "Failed to register {0} with uPnP IGD: {1}"
logger.error("TrSv", msg.format(port, str(e)))
def generate(name, domain, country, state, locale, email, keytype, keylength):
"""Generate a self-signed SSL/TLS certificate."""
if not domain:
logger.error(
"ctl:info:generate", "Choose a fully-qualified domain name of the "
"certificate. Must match a domain present on the system")
domain = click.prompt("Domain name")
if not country:
logger.info("ctl:cert:generate",
"Two-character country code (ex.: 'US' or 'CA')")
country = click.prompt("Country code")
if not state:
state = click.prompt("State/Province")
if not locale:
locale = click.prompt("City/Town/Locale")
if not email:
email = click.prompt("Contact email [optional]")
try:
certificates.generate_certificate(name, domain, country, state, locale,
email, keytype, keylength)
except Exception as e:
raise CLIException(str(e))
def composer_install(path, flags={}, env={}):
"""
Install a PHP application bundle via Composer.
:param str path: path to app directory
:param dict flags: command line flags to pass to Composer
:param dict env: command line environment variables
"""
verify_composer()
cwd = os.getcwd()
os.chdir(path)
shell("composer self-update")
cmdflags = []
for x in flags:
flg = "-{0}".format(x) if len(x) == 1 else "--{0}".format(x)
cmdflags.append("{0}={1}".format(flg, flags[x]) if flags[x] else flg)
s = shell("composer install {0}".format(" ".join(cmdflags)), env=env)
os.chdir(cwd)
if s["code"] != 0:
errmsg = "Composer install of {0} failed.".format(path)
logmsg = "Composer install failure details:\n{0}"
logger.error("PHP", logmsg.format(s["stderr"].decode()))
raise errors.OperationFailedError(errmsg)
def check_updates():
updates = []
gpg = gnupg.GPG()
server = config.get("general", "repo_server")
current = config.get("updates", "current_update", 0)
# Fetch updates from registry server
data = api("https://%s/api/v1/updates/%s" % (server, str(current)), crit=True)
for x in data["updates"]:
ustr, u = str(x["tasks"]), json.loads(x["tasks"])
# Get the update signature and test it
sig = api("https://%s/api/v1/signatures/%s" % (server, x["id"]),
returns="raw", crit=True)
with open("/tmp/%s.sig" % x["id"], "w") as f:
f.write(sig)
v = gpg.verify_data("/tmp/%s.sig" % x["id"], ustr)
if v.trust_level == None:
logger.error("Update %s signature verification failed" % x["id"])
break
else:
updates.append({"id": x["id"], "name": x["name"], "date": x["date"],
"info": x["info"], "tasks": u})
storage.updates.set("updates", updates)
return updates
def put(self, id):
data = request.get_json()["network"]
net = network.get_connections(id)
if not id or not net:
abort(404)
if data.get("operation"):
try:
if data["operation"] == "connect":
net.connect()
elif data["operation"] == "disconnect":
net.disconnect()
elif data["operation"] == "enable":
net.enable()
elif data["operation"] == "disable":
net.disable()
else:
abort(422)
except Exception as e:
logger.error("Network", str(e))
return jsonify(errors={"msg": str(e)}), 500
else:
net.config = data["config"]
net.update()
return jsonify(network=net.serialized)
def put(self, id):
data = request.get_json()["filesystem"]
disk = filesystems.get(id)
if not id or not disk:
abort(404)
try:
if data["operation"] == "mount":
if disk.mountpoint:
abort(400)
elif disk.crypt and not data.get("passwd"):
abort(403)
elif data.get("mountpoint"):
disk.mountpoint = data["mountpoint"]
disk.mount(data.get("passwd"))
elif data["operation"] == "umount":
disk.umount()
elif data["operation"] == "enable":
disk.enable()
elif data["operation"] == "disable":
disk.disable()
except Exception as e:
logger.error("Filesystems", str(e))
return jsonify(errors={"msg": str(e)}), 500
return jsonify(filesystem=disk.serialized)
def remove(*mods):
# Removes a set of Python packages from the system.
s = shell("pip2 uninstall %s" % " ".join(x for x in mods))
if s["code"] != 0:
logger.error("Failed to remove %s via PyPI; %s"%(" ".join(x for x in mods),s["stderr"]))
raise Exception("Failed to remove %s via PyPI, check logs for info"%" ".join(x for x in mods))
def scan(verify=True, cry=True):
"""
Search app directory for applications, load them and store metadata.
Also contacts arkOS repo servers to obtain current list of available
apps, and merges in any updates as necessary.
:param bool verify: Verify app dependencies as the apps are scanned
:param bool cry: Raise exception on dependency install failure?
:return: list of Application objects
:rtype: list
"""
signals.emit("apps", "pre_scan")
logger.debug("Apps", "Scanning for applications")
app_dir = config.get("apps", "app_dir")
if not os.path.exists(app_dir):
os.makedirs(app_dir)
pacman.refresh()
logger.debug("Apps", "Getting system/python/ruby installed list")
inst_list = {
"sys": pacman.get_installed(),
"py": python.get_installed(),
"py2": python.get_installed(py2=True),
"rb": ruby.get_installed()
}
# Get paths for installed apps, metadata for available ones
installed_apps = [x for x in os.listdir(app_dir) if not x.startswith(".")]
api_url = ("https://{0}/api/v1/apps"
.format(config.get("general", "repo_server")))
logger.debug("Apps", "Fetching available apps: {0}".format(api_url))
try:
available_apps = api(api_url)
except Exception as e:
available_apps = []
logger.error("Apps", "Could not get available apps from GRM.")
logger.error("Apps", str(e))
if available_apps:
available_apps = available_apps["applications"]
else:
available_apps = []
# Create objects for installed apps with appropriate metadata
for x in installed_apps:
try:
with open(os.path.join(app_dir, x, "manifest.json"), "r") as f:
data = json.loads(f.read())
except ValueError:
warn_str = "Failed to load {0} due to a JSON parsing error"
logger.warning("Apps", warn_str.format(x))
continue
except IOError:
warn_str = "Failed to load {0}: manifest file inaccessible "\
"or not present"
logger.warning("Apps", warn_str.format(x))
continue
logger.debug("Apps", " *** Loading {0}".format(data["id"]))
app = App(**data)
app.installed = True
for y in enumerate(available_apps):
if app.id == y[1]["id"] and app.version != y[1]["version"]:
app.upgradable = y[1]["version"]
if app.id == y[1]["id"]:
app.assets = y[1]["assets"]
available_apps[y[0]]["installed"] = True
app.load(verify=verify, cry=cry, installed=inst_list)
storage.applications[app.id] = app
# Convert available apps payload to objects
for x in available_apps:
if not x.get("installed"):
app = App(**x)
app.installed = False
storage.applications[app.id] = app
if verify:
verify_app_dependencies()
signals.emit("apps", "post_scan")
return storage.applications
def remove(*mods):
# Remove an installed NPM package.
s = shell("npm uninstall %s" % " ".join(x for x in mods), stderr=True)
if s["code"] != 0:
logger.error("Failed to remove %s via npm; log output follows:\n%s"%(" ".join(x for x in mods),s["stderr"]))
raise Exception("Failed to remove %s via npm, check logs for info"%" ".join(x for x in mods))
def install(*mods):
# Install a set of Python packages from PyPI.
s = shell("pip2 install %s" % " ".join(x for x in mods))
if s["code"] != 0:
logger.error("Failed to install %s via PyPI; %s"%(" ".join(x for x in mods),s["stderr"]))
raise Exception("Failed to install %s via PyPI, check logs for info"%" ".join(x for x in mods))
def show(self, file=None):
"""Reimplement."""
if self.message:
logger.error("ctl:exc", self.format_message())