def on_main(self): if not self.import_dir: return if not os.path.isdir(self.import_dir): logging.error("Import directory does not exist. Shutting down.") self.import_path = False self.shutdown(no_restart=True) return db = DB() for import_file in get_files(self.import_dir, exts=self.exts): idec = import_file.base_name try: with import_file.open("rb") as f: f.seek(0, 2) fsize = f.tell() except IOError: logging.debug(f"Import file {import_file.base_name} is busy.") continue if not (import_file.path in self.filesizes and self.filesizes[import_file.path] == fsize): self.filesizes[import_file.path] = fsize logging.debug(f"New file '{import_file.base_name}' detected") continue db.query( """ SELECT meta FROM assets WHERE meta->>%s = %s """, [self.identifier, idec], ) for (meta, ) in db.fetchall(): asset = Asset(meta=meta, db=db) if not (asset["id_storage"] and asset["path"]): mk_error(import_file, "This file has no target path.") continue if self.versioning and os.path.exists(asset.file_path): version_backup(asset) do_import(self, import_file, asset) break else: mk_error(import_file, "This file is not expected.") for fname in os.listdir(self.import_dir): if not fname.endswith(".error.txt"): continue idec = fname.replace(".error.txt", "") if idec not in [ os.path.splitext(f)[0] for f in os.listdir(self.import_dir) ]: os.remove(os.path.join(self.import_dir, fname))
def on_init(self): self.services = {} db = DB() db.query("SELECT id, pid FROM services WHERE host=%s", [config["host"]]) for _, pid in db.fetchall(): if pid: self.kill_service(pid) db.query("UPDATE services SET state = 0 WHERE host=%s", [config["host"]]) db.commit()
def build(self, *args, **kwargs): db = DB() id_user = int(kwargs.get("id_user", 0)) if id_user and self["user"]["is_admin"]: user = User(id_user, db=db) else: user = self["user"] self["user"] = user password = kwargs.get("password", False) full_name = kwargs.get("full_name", False) dashboard = kwargs.get("dashboard", "") user_changed = False if cherrypy.request.method == "POST": if full_name: user["full_name"] = kwargs["full_name"] user_changed = True if password: if len(password) < 8: self.context.message(WEAK_PASS_MSG, "error") return user.set_password(kwargs["password"]) user_changed = True if dashboard != user["dashboard"]: user["dashboard"] = dashboard user_changed = True if user_changed: user.save() if self["user"].id == user.id: self.context["user"] = user.meta self.context.message("User profile saved") db.query("SELECT meta FROM users WHERE meta->>'is_admin' = 'true'") self["admins"] = [User(meta=meta) for meta, in db.fetchall()] self["name"] = "profile" self["title"] = "User profile" self["rights"] = [ ["asset_edit", "Edit assets", "folders"], ["asset_create", "Create assets", "folders"], ["rundown_view", "View rundown", "playout_channels"], ["rundown_edit", "Edit rundown", "playout_channels"], ["scheduler_view", "View scheduler", "playout_channels"], ["scheduler_edit", "Modify scheduler", "playout_channels"], ["job_control", "Control jobs", "actions"], ["service_control", "Control services", "services"], ["mcr", "Control playout", "playout_channels"], ]
def j(*args): print db = DB() db.query(""" SELECT j.id, j.id_action, j.settings, j.priority, j.retries, j.status, j.progress, j.message, j.creation_time, j.start_time, j.end_time, a.meta FROM jobs AS j, assets AS a WHERE a.id = j.id_asset AND j.status in (0,1,5) ORDER BY id DESC LIMIT 50 """) for ( id, id_action, settings, priority, retries, status, progress, message, creation_time, start_time, end_time, meta, ) in db.fetchall(): asset = Asset(meta=meta) line = "{:<30}".format(asset) line += "{} {:.02f}%\n".format(status, progress) try: sys.stdout.write(line) sys.stdout.flush() except IOError: pass
def on_init(self): self.service_type = "conv" self.actions = [] db = DB() db.query(""" SELECT id, title, service_type, settings FROM actions ORDER BY id """) for id_action, title, service_type, settings in db.fetchall(): if service_type == self.service_type: logging.debug(f"Registering action {title}") self.actions.append(Action(id_action, title, xml(settings))) self.reset_jobs()
def get_bin_first_item(id_bin, db=False): if not db: db = DB() db.query( """ SELECT id, meta FROM items WHERE id_bin=%s ORDER BY position LIMIT 1 """, [id_bin], ) for _, meta in db.fetchall(): return Item(meta=meta, db=db) return False
def get_user(login, password, db=False): if not db: db = DB() try: db.query( """ SELECT meta FROM users WHERE login=%s AND password=%s """, [login, get_hash(password)], ) except ValueError: return False res = db.fetchall() if not res: return False return User(meta=res[0][0])
def get_day_events(id_channel, date, num_days=1): chconfig = config["playout_channels"][id_channel] start_time = datestr2ts(date, *chconfig.get("day_start", [6, 0])) end_time = start_time + (3600 * 24 * num_days) db = DB() db.query( """ SELECT id, meta FROM events WHERE id_channel=%s AND start > %s AND start < %s """, (id_channel, start_time, end_time), ) for _, meta in db.fetchall(): yield Event(meta=meta)
def asset_by_path(id_storage, path, db=False): id_storage = str(id_storage) path = path.replace("\\", "/") if not db: db = DB() db.query( """ SELECT id, meta FROM assets WHERE media_type = %s AND meta->>'id_storage' = %s AND meta->>'path' = %s """, [MediaType.FILE, id_storage, path], ) for id, meta in db.fetchall(): return Asset(meta=meta, db=db) return False
def heartbeat(self): db = DB() db.query("SELECT state FROM services WHERE id=%s", [self.id_service]) try: state = db.fetchall()[0][0] except IndexError: state = ServiceState.KILL else: if state == 0: state = 1 db.query( "UPDATE services SET last_seen=%s, state=%s WHERE id=%s", [time.time(), state, self.id_service], ) db.commit() if state in [ ServiceState.STOPPED, ServiceState.STOPPING, ServiceState.KILL ]: self.shutdown()
def reset_jobs(self): db = DB() db.query( """ UPDATE jobs SET id_service=NULL, progress=0, retries=0, status=5, message='Restarting after service restart', start_time=0, end_time=0 WHERE id_service=%s AND STATUS IN (0,1,5) RETURNING id """, [self.id_service], ) for (id_job, ) in db.fetchall(): logging.info(f"Restarting job ID {id_job} (converter restarted)") db.commit()
def api_system(**kwargs): """ Returns system status and controls services Arguments: request list of information to show. defaults to everything ["services", "hosts"] stop stop service by its ID start start service by its ID kill kill service by its ID set_autostart toggle service autostart param (True/False) """ user = kwargs.get("user", anonymous) message = "" if not user: return NebulaResponse(401) db = DB() request = kwargs.get("request", ["services", "hosts"]) if "stop" in kwargs: id_service = kwargs["stop"] if not type(id_service) == int: return NebulaResponse(400, "Invalid ID service to stop") if not user.has_right("service_control", id_service): return NebulaResponse( 403, "You are not allowed to control this service") db.query("UPDATE services SET state=3 WHERE id=%s AND state = 1", [id_service]) db.commit() logging.info(f"{user} requested service ID {id_service} " f"({config['services'][id_service]['title']}) stop") message = "Service is stopping" if "start" in kwargs: id_service = kwargs["start"] if not type(id_service) == int: return NebulaResponse(400, "Invalid ID service to start") if not user.has_right("service_control", id_service): return NebulaResponse( 403, "You are not allowed to control this service") db.query("UPDATE services SET state=2 WHERE id=%s AND state = 0", [id_service]) db.commit() logging.info(f"{user} requested service ID {id_service} " f"({config['services'][id_service]['title']}) start") message = "Service is starting" if "kill" in kwargs: id_service = kwargs["kill"] if not type(id_service) == int: return NebulaResponse(400, "Invalid ID service to kill") if not user.has_right("service_control", id_service): return NebulaResponse( 403, "You are not allowed to control this service") db.query("UPDATE services SET state=4 WHERE id=%s AND state = 3", [id_service]) db.commit() logging.info(f"{user} requested service ID {id_service} " f"({config['services'][id_service]['title']}) kill") message = "Attempting to kill the service" if "autostart" in kwargs: id_service = kwargs["autostart"] if not type(id_service) == int: return NebulaResponse(400, "Invalid ID service to set autostart") if not user.has_right("service_control", id_service): return NebulaResponse( 403, "You are not allowed to control this service") db.query("UPDATE services SET autostart=NOT autostart WHERE id=%s", [id_service]) logging.info(f"{user} requested service ID {id_service} " f"({config['services'][id_service]['title']}) autostart") db.commit() message = "Service auto-start updated" result = {} if "services" in request: services = [] db.query(""" SELECT id, service_type, host, title, autostart, state, last_seen FROM services ORDER BY id ASC """) for id, service_type, host, title, autostart, state, last_seen in db.fetchall( ): service = { "id": id, "type": service_type, "host": host, "title": title, "autostart": autostart, "state": state, "last_seen": last_seen, "last_seen_before": time.time() - last_seen, } services.append(service) result["services"] = services if "hosts" in request: hosts = [] db.query( "SELECT hostname, last_seen, status FROM hosts ORDER BY hostname") for hostname, last_seen, status in db.fetchall(): host = status host["hostname"] = hostname host["last_seen"] = last_seen hosts.append(host) result["hosts"] = hosts return NebulaResponse(200, data=result, message=message)
def on_main(self): db = DB() self.existing = [] start_time = time.time() db.query("SELECT meta FROM assets WHERE media_type=1 AND status=1") for (meta, ) in db.fetchall(): asset = Asset(meta=meta, db=db) file_path = asset.file_path self.existing.append(file_path) duration = time.time() - start_time if duration > 5 or config.get("debug_mode", False): logging.debug(f"Online assets loaded in {s2time(duration)}") start_time = time.time() for wf_settings in self.settings.findall("folder"): id_storage = int(wf_settings.attrib["id_storage"]) rel_wf_path = wf_settings.attrib["path"] quarantine_time = int( wf_settings.attrib.get("quarantine_time", "10")) id_folder = int(wf_settings.attrib.get("id_folder", 12)) storage_path = storages[id_storage].local_path watchfolder_path = os.path.join(storage_path, rel_wf_path) if not os.path.exists(watchfolder_path): logging.warning("Skipping non-existing watchfolder", watchfolder_path) continue i = 0 for file_object in get_files( watchfolder_path, recursive=wf_settings.attrib.get("recursive", False), hidden=wf_settings.attrib.get("hidden", False), case_sensitive_exts=wf_settings.get( "case_sensitive_exts", False), ): i += 1 if i % 100 == 0 and config.get("debug_mode", False): logging.debug("{} files scanned".format(i)) if not file_object.size: continue full_path = file_object.path if full_path in self.existing: continue now = time.time() asset_path = full_path.replace(storage_path, "", 1).lstrip("/") ext = os.path.splitext(asset_path)[1].lstrip(".").lower() if ext not in FileTypes.exts(): continue asset = asset_by_path(id_storage, asset_path, db=db) if asset: self.existing.append(full_path) continue base_name = get_base_name(asset_path) if quarantine_time and now - file_object.mtime < quarantine_time: logging.debug(f"{base_name} is too young. Skipping") continue asset = Asset(db=db) asset["content_type"] = FileTypes.by_ext(ext) asset["media_type"] = MediaType.FILE asset["id_storage"] = id_storage asset["path"] = asset_path asset["ctime"] = now asset["mtime"] = now asset["status"] = ObjectStatus.CREATING asset["id_folder"] = id_folder asset["title"] = base_name asset.load_sidecar_metadata() failed = False for post_script in wf_settings.findall("post"): try: exec(post_script.text) except Exception: log_traceback( f"Error executing post-script on {asset}") failed = True if not failed: asset.save(set_mtime=False) duration = time.time() - start_time if duration > 60 or config.get("debug_mode", False): logging.debug(f"Watchfolders scanned in {s2time(duration)}")
def build(self, *args, **kwargs): db = DB() db.query("SELECT hostname, last_seen, status FROM hosts") metrics = Metrics() for hostname, last_seen, status in db.fetchall(): metrics.add("inactive_seconds", time.time() - last_seen) for name, tags, value in status.get("metrics", []): if not name.startswith("shared_"): tags["hostname"] = hostname metrics.add(name, value, **tags) for user in request_stats: for method in request_stats[user]: metrics.add( "api_requests", request_stats[user][method], user=user, method=method, ) db.query("select status, count(status) from jobs group by status;") for status, count in db.fetchall(): status_label = [ "Pending", "In progress", "Completed", "Failed", "Aborted", "Restart", "Skipped", ][status] metrics.add("jobs", count, status=status, status_label=status_label) db.query( """ SELECT id, service_type, host, title, state, last_seen FROM services """ ) for id, stype, hostname, title, state, last_seen in db.fetchall(): inactive = max(0, int(time.time() - last_seen)) metrics.add( "service_state", state, hostname=hostname, id=id, title=title, service_type=stype, ) metrics.add( "service_inactive_seconds", inactive, hostname=hostname, id=id, title=title, service_type=stype, ) self.is_raw = True self.body = metrics.render(prefix="nebula", site_name=config["site_name"]) self["mime"] = "text/txt"
def build(self, *args, **kwargs): self["name"] = "passreset" self["title"] = "Password reset" self["mode"] = "email-entry" # # REQUEST EMAIL # if "email" in kwargs: email = kwargs["email"].strip() if not re.match(EMAIL_REGEXP, email): self.context.message("Invalid e-mail address specified", "error") return db = DB() db.query( """ SELECT meta FROM users where LOWER(meta->>'email') = LOWER(%s) """, [email], ) try: user = User(meta=db.fetchall()[0][0], db=db) except IndexError: self.context.message("No such user", "error") return if time.time() - user.meta.get("pass_reset_time", 0) < 3600: self.context.message( "Only one password reset request per hour is allowed", "error" ) return token = get_guid() user["pass_reset_time"] = time.time() user["pass_reset_code"] = token mailvars = { "name": user["full_name"] or user["login"], "site_name": config["site_name"], "hub_url": config.get( "hub_url", f"https://{config['site_name']}.nbla.cloud" ), "token": token, } body = MAIL_BODY.format(**mailvars) try: send_mail(email, "Nebula password reset", body) except Exception: log_traceback() self.context.message( """Unable to send password reset email. Please contact your system administrator""", "error", ) return user.save() self["mode"] = "mail-sent" return # # GOT TOKEN # elif "token" in kwargs: token = kwargs["token"].strip() self["mode"] = False self["token"] = token if not re.match(GUID_REGEXP, token): self.context.message("Invalid token specified", "error") return db = DB() db.query( "SELECT meta FROM users WHERE meta->>'pass_reset_code' = %s", [token] ) try: user = User(meta=db.fetchall()[0][0], db=db) except IndexError: self.context.message("No such token", "error") return if user["pass_reset_time"] < time.time() - 3600: self.context.message("Token expired.", "error") self["mode"] = "email-entry" return pass1 = kwargs.get("pass1", False) pass2 = kwargs.get("pass2", False) if pass1 and pass2: if pass1 != pass2: self["mode"] = "pass-entry" self.context.message("Passwords don't match", "error") return if len(pass1) < 8: self["mode"] = "pass-entry" self.context.message( "The password is weak. Must be at least 8 characters", "error" ) return user.set_password(pass1) del user.meta["pass_reset_code"] del user.meta["pass_reset_time"] user.save() self["mode"] = "finished" return self["mode"] = "pass-entry" return
def load(self, id_action): db = DB() db.query("SELECT title, settings FROM actions WHERE id = %s", [id_action]) for title, settings in db.fetchall(): self.data[id_action] = Action(id_action, title, xml(settings))
def main(self): db = DB() db.query( """ SELECT id, title, autostart, state, last_seen FROM services WHERE host=%s """, [config["host"]], ) # # Start / stop service # for id, title, autostart, state, last_seen in db.fetchall(): messaging.send( "service_state", id=id, state=state, autostart=autostart, last_seen=last_seen, last_seen_before=max(0, int(time.time() - last_seen)), ) if state == ServiceState.STARTING: # Start service if id not in self.services.keys(): self.start_service(id, title, db=db) elif state == ServiceState.KILL: # Kill service if id in self.services.keys(): self.kill_service(self.services[id][0].pid) # # Real service state # service_list = [i for i in self.services.keys()] for id_service in service_list: proc, title = self.services[id_service] if proc.poll() is None: continue del self.services[id_service] logging.warning(f"Service ID {id_service} ({title}) terminated") db.query("UPDATE services SET state=0 WHERE id = %s", [id_service]) db.commit() # # Autostart # db.query( """ SELECT id, title, state, autostart FROM services WHERE host=%s AND state=0 AND autostart=true """, [config["host"]], ) for id, title, state, autostart in db.fetchall(): if id not in self.services.keys(): logging.debug(f"AutoStarting service ID {id} ({title})") self.start_service(id, title)
def on_main(self): """ This method checks if the following event should start automatically at given time. It does not handle AUTO playlist advancing """ if not hasattr(self, "controller"): return if hasattr(self.controller, "on_main"): self.controller.on_main() current_item = self.controller.current_item # YES. CURRENT if not current_item: return db = DB() current_event = get_item_event(current_item.id, db=db) if not current_event: logging.warning("Unable to fetch the current event") return db.query( """ SELECT DISTINCT(e.id), e.meta, e.start FROM events AS e, items AS i WHERE e.id_channel = %s AND e.start > %s AND e.start <= %s AND i.id_bin = e.id_magic ORDER BY e.start ASC LIMIT 1 """, [self.id_channel, current_event["start"], time.time()], ) try: next_event = Event(meta=db.fetchall()[0][1], db=db) except IndexError: self.auto_event = False return if self.auto_event == next_event.id: return run_mode = int(next_event["run_mode"]) or RunMode.RUN_AUTO if not run_mode: return elif not next_event.bin.items: return elif run_mode == RunMode.RUN_MANUAL: pass # ????? elif run_mode == RunMode.RUN_SOFT: logging.info("Soft cue", next_event) # if current item is live, take next block/lead out automatically play = self.current_live for i, r in enumerate(current_event.bin.items): if r["item_role"] == "lead_out": try: self.cue( id_channel=self.id_channel, id_item=current_event.bin.items[i + 1].id, db=db, play=play, ) self.auto_event = next_event.id break except IndexError: pass else: try: id_item = next_event.bin.items[0].id except KeyError: id_item = 0 if not self.controller.cued_item: return if id_item != self.controller.cued_item.id: self.cue(id_channel=self.id_channel, id_item=id_item, db=db) self.auto_event = next_event.id return elif run_mode == RunMode.RUN_HARD: logging.info("Hard cue", next_event) id_item = next_event.bin.items[0].id self.cue(id_channel=self.id_channel, id_item=id_item, play=True, db=db) self.auto_event = next_event.id return
def main(self): storages_conf = config.get("storages", "all") db = DB() db.query("SELECT id, settings FROM storages") for id_storage, storage_settings in db.fetchall(): if type(storages_conf) == list and id_storage not in storages_conf: continue storage = Storage(id_storage, **storage_settings) if storage: storage_string = f"{config['site_name']}:{storage.id}" storage_ident_path = os.path.join(storage.local_path, ".nebula_root") if not ( os.path.exists(storage_ident_path) and storage_string in [line.strip() for line in open(storage_ident_path).readlines()] ): try: with open(storage_ident_path, "a") as f: f.write(storage_string + "\n") except Exception: if self.first_run: logging.warning(f"{storage} is mounted, but read only") else: if self.first_run: logging.info(f"{storage} is mounted and root is writable") continue s, i, lcheck = storage_status.get(id_storage, [True, 2, 0]) if not s and time.time() - lcheck < i: continue if s: logging.info(f"{storage} is not mounted. Mounting...") if not os.path.exists(storage.local_path): try: os.mkdir(storage.local_path) except Exception: if s: logging.error(f"Unable to create mountpoint for {storage}") storage_status[id_storage] = [False, 240, time.time()] continue self.mount(storage) if ismount(storage.local_path): logging.goodnews(f"{storage} mounted successfully") if id_storage not in storage_status: storage_status[id_storage] = [True, 2, 0] storage_status[id_storage][0] = True storage_status[id_storage][1] = 2 else: if s: logging.error(f"{storage} mounting failed") storage_status[id_storage][0] = False check_interval = storage_status[id_storage][1] storage_status[id_storage][1] = min(240, check_interval * 2) storage_status[id_storage][2] = time.time()
def build(self, *args, **kwargs): # # If user has custom dashboard, load webtool plugin # custom_dash = self["user"]["dashboard"] if custom_dash: try: Plugin, title = self["site"]["webtools"].tools[custom_dash] except KeyError: raise cherrypy.HTTPError(404, f"No such tool {custom_dash}") try: args = args[1:] except IndexError: args = [] plugin = Plugin(self, custom_dash) self["title"] = title self.view = "tool" body = plugin.build(*args, **kwargs) if plugin.native: self.is_raw = False self["body"] = body else: self.is_raw = True self.body = body return # # Hosts information (node status) # storage_info = {} db = DB() hosts = {} db.query(""" SELECT hostname, last_seen, status FROM hosts ORDER BY hostname ASC """) tagmap = { "cpu_usage": "cpu", "memory_bytes_total": "mem_total", "memory_bytes_free": "mem_free", "swap_bytes_total": "swp_total", "swap_bytes_free": "swp_free", } sinfo = {id: {"title": storages[id].title} for id in storages} for hostname, last_seen, status in db.fetchall(): host_info = {} for name, tags, value in status.get("metrics", []): if name == "storage_bytes_total" and tags.get( "mountpoint") == "/": host_info["root_total"] = value elif name == "storage_bytes_free" and tags.get( "mountpoint") == "/": host_info["root_free"] = value elif name == "shared_storage_bytes_total": sinfo[int(tags.get("id"))]["total"] = value elif name == "shared_storage_bytes_free": sinfo[int(tags.get("id"))]["free"] = value elif name in tagmap: host_info[tagmap[name]] = value hosts[hostname] = host_info storage_info = [{ "id": id, **d } for id, d in sinfo.items() if d.get("total") and d.get("free")] # # MAM statistics # object_counts = {} for obj_type in ["assets", "items", "bins", "events"]: db.query("SELECT COUNT(id) FROM {}".format(obj_type)) object_counts[obj_type] = db.fetchall()[0][0] self["name"] = "dashboard" self["title"] = "Dashboard" self["js"] = [] self["hosts"] = hosts self["storages"] = storage_info self["object_counts"] = object_counts
def run(*args): id_service = args[0] if id_service == "hub": import hub try: hub_instance = hub.CherryAdmin(**hub.hub_config) except Exception: log_traceback() critical_error("Unhandled exception in Hub") return try: id_service = int(id_service) except ValueError: critical_error("Service ID must be integer") db = DB() db.query( """ SELECT service_type, title, host, loop_delay, settings FROM services WHERE id=%s """, [id_service], ) try: agent, title, host, loop_delay, settings = db.fetchall()[0] except IndexError: critical_error( f"Unable to start service {id_service}. No such service") config["user"] = logging.user = title if host != config["host"]: critical_error("This service should not run here.") if settings: try: settings = xml(settings) except Exception: log_traceback() logging.error("Malformed settings XML:\n", settings) db.query("UPDATE services SET autostart=0 WHERE id=%s", [id_service]) db.commit() critical_error("Unable to start service") _module = __import__("services." + agent, globals(), locals(), ["Service"]) Service = _module.Service service = Service(id_service, settings) while True: try: service.on_main() last_run = time.time() while True: time.sleep(min(loop_delay, 2)) service.heartbeat() if time.time() - last_run >= loop_delay: break except (KeyboardInterrupt): logging.warning("Keyboard interrupt") break except (SystemExit): break except Exception: log_traceback() time.sleep(2) sys.exit(1) try: if sys.argv[1] == "once": break except IndexError: pass