def install(self, extra_vars={}, enable=True, nthread=None):
"""
Install reverse proxy, including prep and app recipes.
:param dict extra_vars: Extra form variables as provided by app
:param bool enable: Enable the site in nginx on install?
:param message message: Message object to update with status
"""
if not nthread:
nthread = NotificationThread()
try:
self._install(extra_vars, enable, nthread)
except Exception as e:
nthread.complete(Notification("error", "Webs", str(e)))
raise
def generate_certificate(
id, domain, country, state="", locale="", email="", keytype="RSA",
keylength=2048, dhparams="/etc/arkos/ssl/dh_params.pem",
nthread=NotificationThread()):
"""
Generate and save a new self-signed certificate.
If this domain has no prior self-signed certificates, a new
CertificateAuthority is also generated to sign this certificate.
:param str id: Name to assign certificate
:param str domain: Domain name to associate with (subject CN)
:param str country: Two-letter country code (e.g. 'US' or 'CA')
:param str state: State or province
:param str locale: City, town or locale
:param str email: Contact email for user
:param str keytype: Key type. One of "RSA" or "DSA"
:param int keylength: Key length. 2048, 4096, etc.
:param str dhparams: Path to dh_params file on disk
:param NotificationThread nthread: notification thread to use
:returns: Certificate that was generated
:rtype: Certificate
"""
try:
return _generate_certificate(
id, domain, country, state, locale, email, keytype, keylength,
dhparams, nthread)
except Exception as e:
nthread.complete(Notification("error", "Certificates", str(e)))
raise
def create(id, data=True, nthread=NotificationThread()):
"""
Convenience function to create a backup.
:param str id: ID of associated app (or website) to backup
:param bool data: Backup app data also?
:returns: Backup info
:rtype: Backup
"""
controller = None
if id == "arkOS":
controller = arkOSBackupCfg("arkOS", "setting",
version=arkos_version)
return controller.backup()
app = applications.get(id)
if app and app.type != "website" and hasattr(app, "_backup"):
controller = app._backup(app.id, app.icon, version=app.version)
else:
sites = websites.get()
for x in sites:
if x.id == id:
controller = x.backup
break
if not controller:
raise errors.InvalidConfigError("No backup controller found")
return controller.backup(data=data, nthread=nthread)
def restore(backup, data=True, nthread=NotificationThread()):
"""
Convenience function to restore a backup.
:param Backup backup: Backup to restore
:param bool data: Restore included data files as well?
:returns: Backup info
:rtype: Backup
"""
controller = None
if backup["type"] == "site":
sites = websites.get()
for x in sites:
if x.id == backup["pid"]:
controller = x.backup
break
else:
app = applications.get(backup["site_type"])
controller = app._backup(backup["pid"], backup["icon"], True)
else:
app = applications.get(backup["pid"])
controller = app._backup()
if not controller:
raise errors.InvalidConfigError("No backup controller found")
b = controller.restore(backup, data, nthread)
return b
def uninstall(self, force=False, nthread=NotificationThread()):
"""
Uninstall the arkOS application from the system.
:param bool force: Uninstall the app even if others depend on it?
:param NotificationThread nthread: notification thread to use
"""
signals.emit("apps", "pre_remove", self)
msg = "Uninstalling application..."
nthread.update(Notification("info", "Apps", msg))
exclude = ["openssl", "openssh", "nginx", "python2", "git",
"nodejs", "npm"]
# Make sure this app can be successfully removed, and if so also remove
# any system-level packages that *only* this app requires
for x in get(installed=True):
for item in x.dependencies:
if item["type"] == "app" and item["package"] == self.id \
and not force:
exc_str = "{0} depends on this application"
raise errors.InvalidConfigError(exc_str.format(x.name))
elif item["type"] == "system":
exclude.append(item["package"])
# Stop any running services associated with this app
for item in self.dependencies:
if item["type"] == "system" and not item["package"] in exclude:
if item.get("daemon"):
try:
services.get(item["daemon"]).stop()
services.get(item["daemon"]).disable()
except:
pass
pacman.remove([item["package"]],
purge=config.get("apps", "purge"))
# Remove the app's directory and cleanup the app object
shutil.rmtree(os.path.join(config.get("apps", "app_dir"), self.id))
self.loadable = False
self.installed = False
# Regenerate the firewall and re-block the abandoned ports
regen_fw = False
for x in self.services:
if x["ports"]:
regen_fw = True
if regen_fw:
tracked_services.deregister(self.id)
ports = []
for s in self.services:
if s.get("default_policy", 0) and s["ports"]:
ports.append(s["ports"])
if ports and config.get("general", "enable_upnp"):
tracked_services.close_all_upnp(ports)
smsg = "{0} uninstalled successfully".format(self.name)
nthread.complete(Notification("success", "Apps", smsg))
signals.emit("apps", "post_remove", self)
def _request_acme(self, job, data):
nthread = NotificationThread(id=job.id)
try:
cert = certificates.request_acme_certificate(
data["domain"], nthread=nthread)
except:
remove_record("certificate", data["id"])
raise
else:
push_record("certificate", cert.serialized)
def remove(self, nthread=NotificationThread()):
"""
Remove website, including prep and app recipes.
:param message message: Message object to update with status
"""
try:
self._remove(nthread)
except Exception as e:
nthread.complete(Notification("error", "Webs", str(e)))
raise
def request_acme_certificate(domain, webroot="", nthread=NotificationThread()):
"""
Request, validate and save a new ACME certificate from Let's Encrypt CA.
:param str domain: Domain name to associate with (subject CN)
:param str webroot: Path to root of web directory, to place .well-known
:param NotificationThread nthread: notification thread to use
"""
try:
return _request_acme_certificate(domain, webroot, nthread)
except Exception as e:
nthread.complete(Notification("error", "Certificates", str(e)))
raise
def _post(self, job, data):
nthread = NotificationThread(id=job.id)
sapp = applications.get(data["app"])
site = sapp._website
site = site(sapp, data["id"], data["domain"], data["port"])
try:
specialmsg = site.install(data["extra_data"], True, nthread)
if specialmsg:
Notification("info", "Websites", specialmsg).send()
push_record("website", site.serialized)
except Exception as e:
remove_record("website", data["id"])
raise
def generate_authority(domain, nthread=NotificationThread()):
"""
Generate and save a new certificate authority for signing.
:param str domain: Domain name to use for certificate authority
:returns: Certificate authority
:rtype: CertificateAuthority
"""
try:
return _generate_authority(domain)
except Exception as e:
nthread.complete(Notification("error", "Certificates", str(e)))
raise
def update(self, nthread=NotificationThread()):
"""
Run an update on this website.
Pulls update data from arkOS app package and metadata, and uses it to
update this particular website instance to the latest version.
:param message message: Message object to update with status
"""
try:
self._update(nthread)
except Exception as e:
nthread.complete(Notification("error", "Webs", str(e)))
raise
def create(self,
mount=False,
will_crypt=False,
nthread=NotificationThread()):
"""
Create virtual disk image.
:param bool mount: Mount after creation?
:param bool will_crypt: Will this disk be encrypted later?
:param NotificationThread nthread: notification thread to use
"""
nthread.title = "Creating virtual disk"
vdisk_dir = config.get("filesystems", "vdisk_dir")
if not os.path.exists(vdisk_dir):
os.mkdir(vdisk_dir)
self.path = str(os.path.join(vdisk_dir, self.id + ".img"))
if os.path.exists(self.path):
raise errors.InvalidConfigError("This virtual disk already exists")
# Create an empty file matching disk size
signals.emit("filesystems", "pre_add", self)
msg = "Creating virtual disk..."
nthread.update(Notification("info", "Filesystems", msg))
with open(self.path, "wb") as f:
written = 0
with open("/dev/zero", "rb") as zero:
while self.size > written:
written += 1024
f.write(zero.read(1024))
if not will_crypt:
# Get a free loopback device and mount
loop = losetup.find_unused_loop_device()
loop.mount(str(self.path), offset=1048576)
# Make a filesystem
msg = "Writing filesystem..."
nthread.update(Notification("info", "Filesystems", msg))
s = shell("mkfs.ext4 {0}".format(loop.device))
if s["code"] != 0:
excmsg = "Failed to format loop device: {0}"
raise errors.OperationFailedError(excmsg.format(s["stderr"]))
loop.unmount()
msg = "Virtual disk created successfully"
nthread.complete(Notification("success", "Filesystems", msg))
signals.emit("filesystems", "post_add", self)
if mount:
self.mount()
def install_updates(nthread=NotificationThread()):
"""
Install all available updates from arkOS repo server.
:param message message: Message object to update with status
"""
nthread.title = "Installing updates"
updates = storage.updates
if not updates:
return
signals.emit("updates", "pre_install")
amount = len(updates)
responses, ids = [], []
for z in enumerate(updates.values()):
msg = "{0} of {1}...".format(z[0] + 1, amount)
nthread.update(Notification("info", "Updates", msg))
for x in sorted(z[1]["tasks"], key=lambda y: y["step"]):
if x["unit"] == "shell":
s = shell(x["order"], stdin=x.get("data", None))
if s["code"] != 0:
responses.append((x["step"], s["stderr"]))
break
elif x["unit"] == "fetch":
try:
download(x["order"], x["data"], True)
except Exception as e:
code = getattr(e, "code", 1)
responses.append((x["step"], str(code)))
break
else:
ids.append(z[1]["id"])
config.set("updates", "current_update", z[1]["id"])
config.save()
continue
for x in responses:
nthread.update(Notification("debug", "Updates", x))
msg = "Installation of update {0} failed. See logs for details."
msg = msg.format(z[1]["id"])
nthread.complete(Notification("error", "Updates", msg))
break
else:
signals.emit("updates", "post_install")
for x in responses:
nthread.update(Notification("debug", "Updates", x))
msg = "Please restart your system for the updates to take effect."
nthread.complete(Notification("success", "Updates", msg))
return ids
def install(self, install_deps=True, load=True, force=False,
cry=False, nthread=NotificationThread()):
"""
Install the arkOS application to the system.
:param bool install_deps: Install the app's dependencies too?
:param bool load: Load the app after install?
:param bool force: Force reinstall if app is already installed?
:param bool cry: Raise exception on dependency install failure?
:param NotificationThread nthread: notification thread to use
"""
try:
self._install(install_deps, load, force, cry, nthread)
except Exception as e:
nthread.complete(Notification("error", "Apps", str(e)))
raise
def install(self,
extra_vars={},
enable=True,
nthread=NotificationThread()):
"""
Install site, including prep and app recipes.
:param dict extra_vars: Extra form variables as provided by client
:param bool enable: Enable the site in nginx on install?
:param message message: Message object to update with status
:returns: special message to the user from app post-install hook (opt)
"""
try:
self._install(extra_vars, enable, nthread)
except Exception as e:
self.clean_up()
nthread.complete(Notification("error", "Webs", str(e)))
raise
def install(job, to_install):
errors = False
nthread = NotificationThread(id=job.id)
nthread.title = "Setting up your server..."
for x in to_install:
a = applications.get(x)
msg = "Installing {0}...".format(x)
nthread.update(Notification("info", "FirstRun", msg))
try:
a.install()
push_record("app", a.serialized)
except:
errors = True
if to_install:
if errors:
msg = ("One or more applications failed to install. "
"Check the App Store pane for more information.")
nthread.complete(Notification("warning", "FirstRun", msg))
else:
msg = ("You may need to restart your device before "
"changes will take effect.")
nthread.complete(Notification("success", "FirstRun", msg))
def _generate(self, job, data):
nthread = NotificationThread(id=job.id)
try:
cert = certificates.generate_certificate(
data["id"], data["domain"], data["country"], data["state"],
data["locale"], data["email"], data["keytype"],
data["keylength"], nthread=nthread)
except:
remove_record("certificate", data["id"])
raise
else:
push_record("certificate", cert.serialized)
try:
basehost = ".".join(data["domain"].split(".")[-2:])
ca = certificates.get_authorities(basehost)
except:
pass
else:
push_record("authority", ca.serialized)
def _post(self, job, data):
nthread = NotificationThread(id=job.id)
disk = filesystems.VirtualDisk(id=data["id"], size=data["size"])
disk.create(will_crypt=data["crypt"], nthread=nthread)
if data["crypt"]:
try:
msg = "Encrypting virtual disk..."
nthread.update(Notification("info", "Filesystems", msg))
disk.encrypt(data["passwd"])
except Exception as e:
disk.remove()
raise
msg = "Virtual disk created successfully"
nthread.complete(Notification("success", "Filesystems", msg))
push_record("filesystem", disk.serialized)
def post(self):
msg = request.get_json()["notification"]
if not msg.get("message") or not msg.get("level")\
or not msg.get("comp"):
abort(400)
notif = Notification(msg["level"], msg["comp"], msg["message"],
msg.get("cls"), msg.get("title"))
# If ID is provided at POST, assume part of thread
if msg.get("id"):
nthread = NotificationThread(id=msg["id"])
if msg.get("complete"):
nthread.complete(notif)
else:
nthread.update(notif)
msg["message_id"] = notif.message_id
else:
notif.send()
return jsonify(notification=msg), 201
def _upload(self, job, name, files):
nthread = NotificationThread(id=job.id)
cert = certificates.upload_certificate(
name, files[0], files[1], files[2], nthread)
push_record("certificate", cert.serialized)
def _install(self, job, app):
nthread = NotificationThread(id=job.id)
app.install(nthread=nthread, force=True, cry=False)
push_record("app", app.serialized)
def _uninstall(self, job, app):
nthread = NotificationThread(id=job.id)
app.uninstall(nthread=nthread)
push_record("app", app.serialized)
def backup(self, data=True, backup_location="",
nthread=NotificationThread()):
"""
Initiate a backup of the associated arkOS app.
:param bool data: Include specified data files in the backup?
:param str backup_location: Save output archive to custom path
:param NotificationThread nthread: notification thread to use
:returns: ``Backup``
:rtype: dict
"""
nthread.title = "Creating a backup"
if not backup_location:
backup_location = config.get("backups", "location")
if self.ctype == "site":
self.version = self.site.app.version
signals.emit("backups", "pre_backup", self)
msg = "Running pre-backup for {0}...".format(self.id)
nthread.update(Notification("info", "Backup", msg))
# Trigger the pre-backup hook for the app/site
if self.ctype == "site":
self.pre_backup(self.site)
else:
self.pre_backup()
# Create backup directory in storage
backup_dir = os.path.join(backup_location, self.id)
try:
os.makedirs(backup_dir)
except:
pass
# Gather config and data file paths to archive
myconfig = self._get_config()
data = self._get_data() if data else []
timestamp = systemtime.get_serial_time()
isotime = systemtime.get_iso_time(timestamp)
archive_name = "{0}-{1}.tar.gz".format(self.id, timestamp)
path = os.path.join(backup_dir, archive_name)
# Zip up the gathered file paths
nthread.complete(Notification("info", "Backup", "Creating archive..."))
with tarfile.open(path, "w:gz") as t:
for f in myconfig+data:
for x in glob.glob(f):
t.add(x)
if self.ctype == "site" and self.site.db:
dbsql = io.StringIO(self.site.db.dump())
dinfo = tarfile.TarInfo(name="/{0}.sql".format(self.site.id))
dinfo.size = len(dbsql.buf)
t.addfile(tarinfo=dinfo, fileobj=dbsql)
# Create a metadata file to track information
info = {"pid": self.id, "type": self.ctype, "icon": self.icon,
"version": self.version, "time": isotime}
if self.site:
info["site_type"] = self.site.app.id
filename = "{0}-{1}.meta".format(self.id, timestamp)
with open(os.path.join(backup_dir, filename), "w") as f:
f.write(json.dumps(info))
# Trigger post-backup hook for the app/site
msg = "Running post-backup for {0}...".format(self.id)
nthread.update(Notification("info", "Backup", msg))
if self.ctype == "site":
self.post_backup(self.site)
else:
self.post_backup()
signals.emit("backups", "post_backup", self)
msg = "{0} backed up successfully.".format(self.id)
nthread.complete(Notification("info", "Backup", msg))
return {"id": "{0}/{1}".format(self.id, timestamp), "pid": self.id,
"path": path, "icon": self.icon, "type": self.ctype,
"time": isotime, "version": self.version,
"size": os.path.getsize(path), "is_ready": True,
"site_type": self.site.app.id if self.site else None}
def _operation(self, job, install, remove):
if install:
try:
pacman.refresh()
prereqs = pacman.needs_for(install)
upgr = (x["id"] for x in pacman.get_installed()
if x.get("upgradable"))
if sorted(upgr) == sorted(install):
# Upgrade
msg = "Performing system upgrade..."
msg = Notification("info", "Packages", msg)
nthread = NotificationThread(id=job.id, message=msg)
pacman.upgrade()
else:
# Install
title = "Installing {0} package(s)".format(len(prereqs))
msg = Notification("info", "Packages", ", ".join(prereqs))
nthread = NotificationThread(id=job.id,
title=title,
message=msg)
pacman.install(install)
for x in prereqs:
try:
info = process_info(pacman.get_info(x))
if "installed" not in info:
info["installed"] = True
push_record("package", info)
except:
pass
except Exception as e:
nthread.complete(Notification("error", "Packages", str(e)))
return
if remove:
try:
prereqs = pacman.depends_for(remove)
title = "Removing {0} package(s)".format(len(prereqs))
msg = Notification("info", "Packages", ", ".join(prereqs))
nthread = NotificationThread(id=job.id,
title=title,
message=msg)
pacman.remove(remove)
for x in prereqs:
try:
info = process_info(pacman.get_info(x))
if "installed" not in info:
info["installed"] = False
push_record("package", info)
except:
pass
except Exception as e:
nthread.complete(Notification("error", "Packages", str(e)))
return
msg = "Operations completed successfully"
nthread.complete(Notification("success", "Packages", msg))
def upload_certificate(
id, cert, key, chain="", dhparams="/etc/arkos/ssl/dh_params.pem",
nthread=NotificationThread()):
"""
Create and save a new certificate from an external file.
:param str id: Name to assign certificate
:param str cert: Certificate as string (PEM format)
:param str key: Key as string (PEM format)
:param str chain: Chain as string (PEM format)
:param NotificationThread nthread: notification thread to use
:returns: Certificate that was imported
:rtype: Certificate
"""
nthread.title = "Uploading TLS certificate"
# Test the certificates are valid
crt = x509.load_pem_x509_certificate(cert, default_backend())
ky = serialization.load_pem_private_key(
key,
password=None,
backend=default_backend()
)
signals.emit("certificates", "pre_add", id)
# Check to see that we have DH params, if not then do that too
if not os.path.exists(dhparams):
msg = "Generating Diffie-Hellman parameters..."
nthread.update(Notification("info", "Certificates", msg))
generate_dh_params(dhparams)
# Create actual certificate object
msg = "Importing certificate..."
nthread.update(Notification("info", "Certificates", msg))
cert_dir = config.get("certificates", "cert_dir")
key_dir = config.get("certificates", "key_dir")
sha1 = binascii.hexlify(crt.fingerprint(hashes.SHA1())).decode()
md5 = binascii.hexlify(crt.fingerprint(hashes.MD5())).decode()
kt = "RSA" if isinstance(ky.public_key(), rsa.RSAPublicKey) else "DSA"
common_name = crt.subject.get_attributes_for_oid(NameOID.COMMON_NAME)
c = Certificate(id=id,
cert_path=os.path.join(cert_dir, "{0}.crt".format(id)),
key_path=os.path.join(key_dir, "{0}.key".format(id)),
keytype=kt, keylength=ky.key_size,
domain=common_name, expiry=crt.not_valid_after,
sha1=sha1, md5=md5)
# Save certificate, key and chainfile (if applicable) to files
# and set perms
with open(c.cert_path, "wb") as f:
f.write(cert)
if chain:
f.write("\n") if not cert.endswith("\n") else None
f.write(chain)
with open(c.key_path, "wb") as f:
f.write(key)
os.chown(c.cert_path, -1, gid)
os.chmod(c.cert_path, 0o660)
os.chown(c.key_path, -1, gid)
os.chmod(c.key_path, 0o660)
storage.certificates[c.id] = c
signals.emit("certificates", "post_add", c)
msg = "Certificate imported successfully"
nthread.complete(Notification("success", "Certificates", msg))
return c
def restore(self, backup, data=True, nthread=NotificationThread()):
"""
Restore an associated arkOS app backup.
:param Backup backup: backup to restore
:param bool data: Restore backed up data files too?
:param NotificationThread nthread: notification thread to use
:returns: ``Backup``
:rtype: dict
"""
nthread.title = "Restoring backup"
# Trigger pre-restore hook for the app/site
signals.emit("backups", "pre_restore", self)
msg = "Running pre-restore for {0}...".format(backup["pid"])
nthread.update(Notification("info", "Backup", msg))
self.pre_restore()
# Extract all files in archive
sitename = ""
nthread.update(Notification("info", "Backup", "Extracting files..."))
with tarfile.open(backup["path"], "r:gz") as t:
for x in t.getnames():
if x.startswith("etc/nginx/sites-available"):
sitename = os.path.basename(x)
t.extractall("/")
# If it's a website that had a database, restore DB via SQL file too
dbpasswd = ""
if self.ctype == "site" and sitename:
self.site = websites.get(sitename)
if not self.site:
websites.scan()
self.site = websites.get(sitename)
meta = configparser.SafeConfigParser()
meta.read(os.path.join(self.site.path, ".arkos"))
sql_path = "/{0}.sql".format(sitename)
if meta.get("website", "dbengine", fallback=None) \
and os.path.exists(sql_path):
nthread.update(
Notification("info", "Backup", "Restoring database..."))
dbmgr = databases.get_managers(meta.get("website", "dbengine"))
if databases.get(sitename):
databases.get(sitename).remove()
db = dbmgr.add_db(sitename)
with open(sql_path, "r") as f:
db.execute(f.read())
os.unlink(sql_path)
if dbmgr.meta.database_multiuser:
dbpasswd = random_string(16)
dbuser = databases.get_users(sitename)
if dbuser:
dbuser.remove()
db_user = dbmgr.add_user(sitename, dbpasswd)
db_user.chperm("grant", db)
# Trigger post-restore hook for the app/site
msg = "Running post-restore for {0}...".format(backup["pid"])
nthread.update(Notification("info", "Backup", msg))
if self.ctype == "site":
self.post_restore(self.site, dbpasswd)
self.site.nginx_enable()
else:
self.post_restore()
signals.emit("backups", "post_restore", self)
backup["is_ready"] = True
msg = "{0} restored successfully.".format(backup["pid"])
nthread.complete(Notification("info", "Backup", msg))
return backup
def _put(self, job, data):
nthread = NotificationThread(id=job.id)
b = backup.restore(data, nthread=nthread)
push_record("backup", b)
def _post(self, job, id):
nthread = NotificationThread(id=job.id)
b = backup.create(id, nthread=nthread)
push_record("backups", b)
def _delete(self, job, id):
nthread = NotificationThread(id=job.id)
site = websites.get(id)
site.remove(nthread)
remove_record("website", id)
remove_record("policy", id)
def _put(self, job, site):
nthread = NotificationThread(id=job.id)
site.update(nthread=nthread)
push_record("website", site.serialized)