def build(self, *args, **kwargs):
services = []
db = DB()
db.query("""
SELECT id, service_type, host, title, autostart, state, last_seen
FROM services ORDER BY id
""")
for id, service_type, host, title, autostart, state, last_seen in db.fetchall(
):
service = {
"id": id,
"service_type": service_type,
"host": host,
"title": title,
"autostart": autostart,
"state": state,
"last_seen": last_seen,
"message": "",
}
if time.time() - last_seen > 120:
nrtime = s2words(time.time() - last_seen)
service["message"] = f"Not responding for {nrtime}"
services.append(service)
self["name"] = "services"
self["title"] = "Services"
self["js"] = ["/static/js/services.js"]
self["data"] = services
def channel_recover(self):
logging.warning("Performing recovery")
db = DB()
db.query(
"""
SELECT id_item, start FROM asrun
WHERE id_channel = %s ORDER BY id DESC LIMIT 1
""",
[self.id_channel],
)
try:
last_id_item, last_start = db.fetchall()[0]
except IndexError:
logging.error("Unable to perform recovery.")
last_item = Item(last_id_item, db=db)
last_item.asset
self.controller.current_item = last_item
self.controller.cued_item = False
self.controller.cued_fname = False
if last_start + last_item.duration <= time.time():
logging.info(f"Last {last_item} has been broadcasted.")
new_item = self.cue_next(item=last_item, db=db, play=True)
else:
logging.info(f"Last {last_item} has not been fully broadcasted.")
new_item = self.cue_next(item=last_item, db=db)
if not new_item:
logging.error("Recovery failed. Unable to cue")
return
self.on_change()
def on_init(self):
self.actions = []
db = DB()
db.query("SELECT id, title, settings FROM actions")
for id, title, settings in db.fetchall():
settings = xml(settings)
self.actions.append(Action(id, title, settings))
def on_main(self):
db = DB()
db.query("SELECT id, meta FROM assets WHERE status=%s",
[ObjectStatus.ONLINE])
for _, meta in db.fetchall():
asset = Asset(meta=meta, db=db)
self.proc(asset)
def passwd(*args):
print()
try:
login = input("Login: "******"Password (will be echoed): ").strip()
is_admin = input("Admin (yes/no): ").strip()
except KeyboardInterrupt:
print()
logging.warning("Interrupted by user")
sys.exit(0)
db = DB()
db.query("SELECT id FROM users WHERE login=%s", [login])
res = db.fetchall()
if not res:
critical_error("Unable to set password: no such user")
u = User(res[0][0], db=db)
if login:
u["login"] = u["full_name"] = login
u["is_admin"] = 1 if is_admin == "yes" else 0
u.set_password(password)
u.save()
print()
logging.goodnews("Password changed")
def meta_exists(key, value, db=False):
if not db:
db = DB()
db.query("SELECT id, meta FROM assets WHERE meta->>%s = %s",
[str(key), str(value)])
for _, meta in db.fetchall():
return Asset(meta=meta, db=db)
return False
def shutdown(self, no_restart=False):
logging.info("Shutting down")
if no_restart:
db = DB()
db.query("UPDATE services SET autostart=FALSE WHERE id=%s",
[self.id_service])
db.commit()
self.on_shutdown()
sys.exit(0)
def on_main(self):
if not self.import_dir:
return
if not os.path.isdir(self.import_dir):
logging.error("Import directory does not exist. Shutting down.")
self.import_path = False
self.shutdown(no_restart=True)
return
db = DB()
for import_file in get_files(self.import_dir, exts=self.exts):
idec = import_file.base_name
try:
with import_file.open("rb") as f:
f.seek(0, 2)
fsize = f.tell()
except IOError:
logging.debug(f"Import file {import_file.base_name} is busy.")
continue
if not (import_file.path in self.filesizes
and self.filesizes[import_file.path] == fsize):
self.filesizes[import_file.path] = fsize
logging.debug(f"New file '{import_file.base_name}' detected")
continue
db.query(
"""
SELECT meta FROM assets
WHERE meta->>%s = %s
""",
[self.identifier, idec],
)
for (meta, ) in db.fetchall():
asset = Asset(meta=meta, db=db)
if not (asset["id_storage"] and asset["path"]):
mk_error(import_file, "This file has no target path.")
continue
if self.versioning and os.path.exists(asset.file_path):
version_backup(asset)
do_import(self, import_file, asset)
break
else:
mk_error(import_file, "This file is not expected.")
for fname in os.listdir(self.import_dir):
if not fname.endswith(".error.txt"):
continue
idec = fname.replace(".error.txt", "")
if idec not in [
os.path.splitext(f)[0] for f in os.listdir(self.import_dir)
]:
os.remove(os.path.join(self.import_dir, fname))
def on_init(self):
self.services = {}
db = DB()
db.query("SELECT id, pid FROM services WHERE host=%s", [config["host"]])
for _, pid in db.fetchall():
if pid:
self.kill_service(pid)
db.query("UPDATE services SET state = 0 WHERE host=%s", [config["host"]])
db.commit()
def build(self, *args, **kwargs):
db = DB()
id_user = int(kwargs.get("id_user", 0))
if id_user and self["user"]["is_admin"]:
user = User(id_user, db=db)
else:
user = self["user"]
self["user"] = user
password = kwargs.get("password", False)
full_name = kwargs.get("full_name", False)
dashboard = kwargs.get("dashboard", "")
user_changed = False
if cherrypy.request.method == "POST":
if full_name:
user["full_name"] = kwargs["full_name"]
user_changed = True
if password:
if len(password) < 8:
self.context.message(WEAK_PASS_MSG, "error")
return
user.set_password(kwargs["password"])
user_changed = True
if dashboard != user["dashboard"]:
user["dashboard"] = dashboard
user_changed = True
if user_changed:
user.save()
if self["user"].id == user.id:
self.context["user"] = user.meta
self.context.message("User profile saved")
db.query("SELECT meta FROM users WHERE meta->>'is_admin' = 'true'")
self["admins"] = [User(meta=meta) for meta, in db.fetchall()]
self["name"] = "profile"
self["title"] = "User profile"
self["rights"] = [
["asset_edit", "Edit assets", "folders"],
["asset_create", "Create assets", "folders"],
["rundown_view", "View rundown", "playout_channels"],
["rundown_edit", "Edit rundown", "playout_channels"],
["scheduler_view", "View scheduler", "playout_channels"],
["scheduler_edit", "Modify scheduler", "playout_channels"],
["job_control", "Control jobs", "actions"],
["service_control", "Control services", "services"],
["mcr", "Control playout", "playout_channels"],
]
def j(*args):
print
db = DB()
db.query("""
SELECT
j.id,
j.id_action,
j.settings,
j.priority,
j.retries,
j.status,
j.progress,
j.message,
j.creation_time,
j.start_time,
j.end_time,
a.meta
FROM
jobs AS j,
assets AS a
WHERE
a.id = j.id_asset
AND j.status in (0,1,5)
ORDER BY
id DESC LIMIT 50
""")
for (
id,
id_action,
settings,
priority,
retries,
status,
progress,
message,
creation_time,
start_time,
end_time,
meta,
) in db.fetchall():
asset = Asset(meta=meta)
line = "{:<30}".format(asset)
line += "{} {:.02f}%\n".format(status, progress)
try:
sys.stdout.write(line)
sys.stdout.flush()
except IOError:
pass
def on_init(self):
self.service_type = "conv"
self.actions = []
db = DB()
db.query("""
SELECT id, title, service_type, settings
FROM actions ORDER BY id
""")
for id_action, title, service_type, settings in db.fetchall():
if service_type == self.service_type:
logging.debug(f"Registering action {title}")
self.actions.append(Action(id_action, title, xml(settings)))
self.reset_jobs()
def get_bin_first_item(id_bin, db=False):
if not db:
db = DB()
db.query(
"""
SELECT id, meta FROM items
WHERE id_bin=%s
ORDER BY position LIMIT 1
""",
[id_bin],
)
for _, meta in db.fetchall():
return Item(meta=meta, db=db)
return False
def on_init(self):
self.last_update = 0
db = DB()
try:
db.query(
"""
INSERT INTO hosts(hostname, last_seen)
VALUES (%s, %s) ON CONFLICT DO NOTHING
""",
[config["host"], time.time()],
)
except psycopg2.IntegrityError:
pass
else:
db.commit()
def asset_by_path(id_storage, path, db=False):
id_storage = str(id_storage)
path = path.replace("\\", "/")
if not db:
db = DB()
db.query(
"""
SELECT id, meta FROM assets
WHERE media_type = %s
AND meta->>'id_storage' = %s
AND meta->>'path' = %s
""",
[MediaType.FILE, id_storage, path],
)
for id, meta in db.fetchall():
return Asset(meta=meta, db=db)
return False
def get_day_events(id_channel, date, num_days=1):
chconfig = config["playout_channels"][id_channel]
start_time = datestr2ts(date, *chconfig.get("day_start", [6, 0]))
end_time = start_time + (3600 * 24 * num_days)
db = DB()
db.query(
"""
SELECT id, meta
FROM events
WHERE id_channel=%s
AND start > %s
AND start < %s
""",
(id_channel, start_time, end_time),
)
for _, meta in db.fetchall():
yield Event(meta=meta)
def get_user(login, password, db=False):
if not db:
db = DB()
try:
db.query(
"""
SELECT meta FROM users
WHERE login=%s AND password=%s
""",
[login, get_hash(password)],
)
except ValueError:
return False
res = db.fetchall()
if not res:
return False
return User(meta=res[0][0])
def heartbeat(self):
db = DB()
db.query("SELECT state FROM services WHERE id=%s", [self.id_service])
try:
state = db.fetchall()[0][0]
except IndexError:
state = ServiceState.KILL
else:
if state == 0:
state = 1
db.query(
"UPDATE services SET last_seen=%s, state=%s WHERE id=%s",
[time.time(), state, self.id_service],
)
db.commit()
if state in [
ServiceState.STOPPED, ServiceState.STOPPING, ServiceState.KILL
]:
self.shutdown()
def reset_jobs(self):
db = DB()
db.query(
"""
UPDATE jobs SET
id_service=NULL,
progress=0,
retries=0,
status=5,
message='Restarting after service restart',
start_time=0,
end_time=0
WHERE
id_service=%s AND STATUS IN (0,1,5)
RETURNING id
""",
[self.id_service],
)
for (id_job, ) in db.fetchall():
logging.info(f"Restarting job ID {id_job} (converter restarted)")
db.commit()
def __init__(self, id_service, settings=False):
logging.debug(f"Initializing service ID {id_service}")
self.id_service = id_service
self.settings = settings
config["id_service"] = id_service
try:
self.on_init()
except SystemExit:
sys.exit(1)
except Exception:
log_traceback(f"Unable to initialize service ID {id_service}")
self.shutdown()
else:
db = DB()
db.query(
"UPDATE services SET last_seen = %s, state=1 WHERE id=%s",
[time.time(), self.id_service],
)
db.commit()
logging.goodnews("Service started")
def on_change(self):
if not self.controller.current_item:
return
item = self.controller.current_item
db = DB()
self.current_asset = item.asset or Asset()
self.current_event = item.event or Event()
logging.info(f"Advanced to {item}")
if self.last_run:
db.query(
"""
UPDATE asrun SET stop = %s
WHERE id = %s""",
[int(time.time()), self.last_run],
)
db.commit()
if self.current_item:
db.query(
"""
INSERT INTO asrun (id_channel, id_item, start)
VALUES (%s, %s, %s)
""",
[self.id_channel, item.id,
time.time()],
)
self.last_run = db.lastid()
db.commit()
else:
self.last_run = False
for plugin in self.plugins:
try:
plugin.on_change()
except Exception:
log_traceback("Plugin on-change failed")
def set_progress(self, progress, message="In progress"):
db = DB()
progress = round(progress, 2)
db.query(
"""
UPDATE jobs SET
status=1,
progress=%s,
message=%s
WHERE id=%s
""",
[progress, message, self.id],
)
db.commit()
messaging.send(
"job_progress",
id=self.id,
id_asset=self.id_asset,
id_action=self.id_action,
status=JobState.IN_PROGRESS,
progress=progress,
message=message,
)
def s(*args):
print()
db = DB()
if len(args) >= 2:
try:
services = tuple([int(i.strip()) for i in args[1:]])
except ValueError:
critical_error("Wrong service ID")
if args[0] == "start":
db.query("UPDATE services SET state=2 WHERE id IN %s AND state=0",
[services])
db.commit()
elif args[0] == "stop":
db.query("UPDATE services SET state=3 WHERE id IN %s AND state=1",
[services])
db.commit()
elif args[0] == "kill":
db.query(
"UPDATE services SET state=4 WHERE id IN %s AND state IN (1,3)",
[services],
)
db.commit()
elif args[0] == "auto":
db.query("UPDATE services SET autostart=TRUE WHERE id IN %s",
[services])
db.commit()
elif args[0] == "noauto":
db.query("UPDATE services SET autostart=FALSE WHERE id IN %s",
[services])
db.commit()
else:
critical_error("Unsupported command: {}".format(args[0]))
time.sleep(1)
show_services(db)
def update_host_info():
hostname = config["host"]
mp2id = {}
for id_storage, storage in storages.items():
mp2id[storage.local_path] = id_storage
p = Promexp(provider_settings={"casparcg": None})
p.metrics.add("runtime_seconds", time.time() - NEBULA_START_TIME)
p.collect()
for metric in list(p.metrics.data.keys()):
mname, tags = metric
if mname in [
"storage_bytes_total", "storage_bytes_free", "storage_usage"
]:
id_storage = mp2id.get(tags["mountpoint"])
if id_storage is None:
continue
value = p.metrics.data[metric]
del p.metrics.data[metric]
p.metrics.add(
f"shared_{mname}",
value,
id=id_storage,
title=storages[id_storage].title,
)
status = {"metrics": p.metrics.dump()}
db = DB()
db.query(
"UPDATE hosts SET last_seen=%s, status=%s WHERE hostname=%s",
[time.time(), json.dumps(status), hostname],
)
db.commit()
def api_system(**kwargs):
"""
Returns system status and controls services
Arguments:
request list of information to show.
defaults to everything ["services", "hosts"]
stop stop service by its ID
start start service by its ID
kill kill service by its ID
set_autostart toggle service autostart param (True/False)
"""
user = kwargs.get("user", anonymous)
message = ""
if not user:
return NebulaResponse(401)
db = DB()
request = kwargs.get("request", ["services", "hosts"])
if "stop" in kwargs:
id_service = kwargs["stop"]
if not type(id_service) == int:
return NebulaResponse(400, "Invalid ID service to stop")
if not user.has_right("service_control", id_service):
return NebulaResponse(
403, "You are not allowed to control this service")
db.query("UPDATE services SET state=3 WHERE id=%s AND state = 1",
[id_service])
db.commit()
logging.info(f"{user} requested service ID {id_service} "
f"({config['services'][id_service]['title']}) stop")
message = "Service is stopping"
if "start" in kwargs:
id_service = kwargs["start"]
if not type(id_service) == int:
return NebulaResponse(400, "Invalid ID service to start")
if not user.has_right("service_control", id_service):
return NebulaResponse(
403, "You are not allowed to control this service")
db.query("UPDATE services SET state=2 WHERE id=%s AND state = 0",
[id_service])
db.commit()
logging.info(f"{user} requested service ID {id_service} "
f"({config['services'][id_service]['title']}) start")
message = "Service is starting"
if "kill" in kwargs:
id_service = kwargs["kill"]
if not type(id_service) == int:
return NebulaResponse(400, "Invalid ID service to kill")
if not user.has_right("service_control", id_service):
return NebulaResponse(
403, "You are not allowed to control this service")
db.query("UPDATE services SET state=4 WHERE id=%s AND state = 3",
[id_service])
db.commit()
logging.info(f"{user} requested service ID {id_service} "
f"({config['services'][id_service]['title']}) kill")
message = "Attempting to kill the service"
if "autostart" in kwargs:
id_service = kwargs["autostart"]
if not type(id_service) == int:
return NebulaResponse(400, "Invalid ID service to set autostart")
if not user.has_right("service_control", id_service):
return NebulaResponse(
403, "You are not allowed to control this service")
db.query("UPDATE services SET autostart=NOT autostart WHERE id=%s",
[id_service])
logging.info(f"{user} requested service ID {id_service} "
f"({config['services'][id_service]['title']}) autostart")
db.commit()
message = "Service auto-start updated"
result = {}
if "services" in request:
services = []
db.query("""
SELECT id, service_type, host, title, autostart, state, last_seen
FROM services ORDER BY id ASC
""")
for id, service_type, host, title, autostart, state, last_seen in db.fetchall(
):
service = {
"id": id,
"type": service_type,
"host": host,
"title": title,
"autostart": autostart,
"state": state,
"last_seen": last_seen,
"last_seen_before": time.time() - last_seen,
}
services.append(service)
result["services"] = services
if "hosts" in request:
hosts = []
db.query(
"SELECT hostname, last_seen, status FROM hosts ORDER BY hostname")
for hostname, last_seen, status in db.fetchall():
host = status
host["hostname"] = hostname
host["last_seen"] = last_seen
hosts.append(host)
result["hosts"] = hosts
return NebulaResponse(200, data=result, message=message)
def soft_stop(self):
logging.info("Soft stop requested")
db = DB()
db.query("UPDATE services SET state=3 WHERE id=%s", [self.id_service])
db.commit()
def main(self):
db = DB()
db.query(
"""
SELECT
id,
title,
autostart,
state,
last_seen
FROM services
WHERE host=%s
""",
[config["host"]],
)
#
# Start / stop service
#
for id, title, autostart, state, last_seen in db.fetchall():
messaging.send(
"service_state",
id=id,
state=state,
autostart=autostart,
last_seen=last_seen,
last_seen_before=max(0, int(time.time() - last_seen)),
)
if state == ServiceState.STARTING: # Start service
if id not in self.services.keys():
self.start_service(id, title, db=db)
elif state == ServiceState.KILL: # Kill service
if id in self.services.keys():
self.kill_service(self.services[id][0].pid)
#
# Real service state
#
service_list = [i for i in self.services.keys()]
for id_service in service_list:
proc, title = self.services[id_service]
if proc.poll() is None:
continue
del self.services[id_service]
logging.warning(f"Service ID {id_service} ({title}) terminated")
db.query("UPDATE services SET state=0 WHERE id = %s", [id_service])
db.commit()
#
# Autostart
#
db.query(
"""
SELECT id, title, state, autostart
FROM services
WHERE host=%s AND state=0 AND autostart=true
""",
[config["host"]],
)
for id, title, state, autostart in db.fetchall():
if id not in self.services.keys():
logging.debug(f"AutoStarting service ID {id} ({title})")
self.start_service(id, title)
def load(self, id_action):
db = DB()
db.query("SELECT title, settings FROM actions WHERE id = %s",
[id_action])
for title, settings in db.fetchall():
self.data[id_action] = Action(id_action, title, xml(settings))
def build(self, *args, **kwargs):
self["name"] = "passreset"
self["title"] = "Password reset"
self["mode"] = "email-entry"
#
# REQUEST EMAIL
#
if "email" in kwargs:
email = kwargs["email"].strip()
if not re.match(EMAIL_REGEXP, email):
self.context.message("Invalid e-mail address specified", "error")
return
db = DB()
db.query(
"""
SELECT meta
FROM users where LOWER(meta->>'email') = LOWER(%s)
""",
[email],
)
try:
user = User(meta=db.fetchall()[0][0], db=db)
except IndexError:
self.context.message("No such user", "error")
return
if time.time() - user.meta.get("pass_reset_time", 0) < 3600:
self.context.message(
"Only one password reset request per hour is allowed", "error"
)
return
token = get_guid()
user["pass_reset_time"] = time.time()
user["pass_reset_code"] = token
mailvars = {
"name": user["full_name"] or user["login"],
"site_name": config["site_name"],
"hub_url": config.get(
"hub_url", f"https://{config['site_name']}.nbla.cloud"
),
"token": token,
}
body = MAIL_BODY.format(**mailvars)
try:
send_mail(email, "Nebula password reset", body)
except Exception:
log_traceback()
self.context.message(
"""Unable to send password reset email.
Please contact your system administrator""",
"error",
)
return
user.save()
self["mode"] = "mail-sent"
return
#
# GOT TOKEN
#
elif "token" in kwargs:
token = kwargs["token"].strip()
self["mode"] = False
self["token"] = token
if not re.match(GUID_REGEXP, token):
self.context.message("Invalid token specified", "error")
return
db = DB()
db.query(
"SELECT meta FROM users WHERE meta->>'pass_reset_code' = %s", [token]
)
try:
user = User(meta=db.fetchall()[0][0], db=db)
except IndexError:
self.context.message("No such token", "error")
return
if user["pass_reset_time"] < time.time() - 3600:
self.context.message("Token expired.", "error")
self["mode"] = "email-entry"
return
pass1 = kwargs.get("pass1", False)
pass2 = kwargs.get("pass2", False)
if pass1 and pass2:
if pass1 != pass2:
self["mode"] = "pass-entry"
self.context.message("Passwords don't match", "error")
return
if len(pass1) < 8:
self["mode"] = "pass-entry"
self.context.message(
"The password is weak. Must be at least 8 characters", "error"
)
return
user.set_password(pass1)
del user.meta["pass_reset_code"]
del user.meta["pass_reset_time"]
user.save()
self["mode"] = "finished"
return
self["mode"] = "pass-entry"
return
def build(self, *args, **kwargs):
#
# If user has custom dashboard, load webtool plugin
#
custom_dash = self["user"]["dashboard"]
if custom_dash:
try:
Plugin, title = self["site"]["webtools"].tools[custom_dash]
except KeyError:
raise cherrypy.HTTPError(404, f"No such tool {custom_dash}")
try:
args = args[1:]
except IndexError:
args = []
plugin = Plugin(self, custom_dash)
self["title"] = title
self.view = "tool"
body = plugin.build(*args, **kwargs)
if plugin.native:
self.is_raw = False
self["body"] = body
else:
self.is_raw = True
self.body = body
return
#
# Hosts information (node status)
#
storage_info = {}
db = DB()
hosts = {}
db.query("""
SELECT hostname, last_seen, status
FROM hosts ORDER BY hostname ASC
""")
tagmap = {
"cpu_usage": "cpu",
"memory_bytes_total": "mem_total",
"memory_bytes_free": "mem_free",
"swap_bytes_total": "swp_total",
"swap_bytes_free": "swp_free",
}
sinfo = {id: {"title": storages[id].title} for id in storages}
for hostname, last_seen, status in db.fetchall():
host_info = {}
for name, tags, value in status.get("metrics", []):
if name == "storage_bytes_total" and tags.get(
"mountpoint") == "/":
host_info["root_total"] = value
elif name == "storage_bytes_free" and tags.get(
"mountpoint") == "/":
host_info["root_free"] = value
elif name == "shared_storage_bytes_total":
sinfo[int(tags.get("id"))]["total"] = value
elif name == "shared_storage_bytes_free":
sinfo[int(tags.get("id"))]["free"] = value
elif name in tagmap:
host_info[tagmap[name]] = value
hosts[hostname] = host_info
storage_info = [{
"id": id,
**d
} for id, d in sinfo.items() if d.get("total") and d.get("free")]
#
# MAM statistics
#
object_counts = {}
for obj_type in ["assets", "items", "bins", "events"]:
db.query("SELECT COUNT(id) FROM {}".format(obj_type))
object_counts[obj_type] = db.fetchall()[0][0]
self["name"] = "dashboard"
self["title"] = "Dashboard"
self["js"] = []
self["hosts"] = hosts
self["storages"] = storage_info
self["object_counts"] = object_counts