def on_init(self):
self.services = {}
db = DB()
db.query("SELECT id, pid FROM services WHERE host=%s", [config["host"]])
for _, pid in db.fetchall():
if pid:
self.kill_service(pid)
db.query("UPDATE services SET state = 0 WHERE host=%s", [config["host"]])
db.commit()
def shutdown(self, no_restart=False):
logging.info("Shutting down")
if no_restart:
db = DB()
db.query("UPDATE services SET autostart=FALSE WHERE id=%s",
[self.id_service])
db.commit()
self.on_shutdown()
sys.exit(0)
def on_init(self):
self.last_update = 0
db = DB()
try:
db.query(
"""
INSERT INTO hosts(hostname, last_seen)
VALUES (%s, %s) ON CONFLICT DO NOTHING
""",
[config["host"], time.time()],
)
except psycopg2.IntegrityError:
pass
else:
db.commit()
def heartbeat(self):
db = DB()
db.query("SELECT state FROM services WHERE id=%s", [self.id_service])
try:
state = db.fetchall()[0][0]
except IndexError:
state = ServiceState.KILL
else:
if state == 0:
state = 1
db.query(
"UPDATE services SET last_seen=%s, state=%s WHERE id=%s",
[time.time(), state, self.id_service],
)
db.commit()
if state in [
ServiceState.STOPPED, ServiceState.STOPPING, ServiceState.KILL
]:
self.shutdown()
def __init__(self, id_service, settings=False):
logging.debug(f"Initializing service ID {id_service}")
self.id_service = id_service
self.settings = settings
config["id_service"] = id_service
try:
self.on_init()
except SystemExit:
sys.exit(1)
except Exception:
log_traceback(f"Unable to initialize service ID {id_service}")
self.shutdown()
else:
db = DB()
db.query(
"UPDATE services SET last_seen = %s, state=1 WHERE id=%s",
[time.time(), self.id_service],
)
db.commit()
logging.goodnews("Service started")
def reset_jobs(self):
db = DB()
db.query(
"""
UPDATE jobs SET
id_service=NULL,
progress=0,
retries=0,
status=5,
message='Restarting after service restart',
start_time=0,
end_time=0
WHERE
id_service=%s AND STATUS IN (0,1,5)
RETURNING id
""",
[self.id_service],
)
for (id_job, ) in db.fetchall():
logging.info(f"Restarting job ID {id_job} (converter restarted)")
db.commit()
def on_change(self):
if not self.controller.current_item:
return
item = self.controller.current_item
db = DB()
self.current_asset = item.asset or Asset()
self.current_event = item.event or Event()
logging.info(f"Advanced to {item}")
if self.last_run:
db.query(
"""
UPDATE asrun SET stop = %s
WHERE id = %s""",
[int(time.time()), self.last_run],
)
db.commit()
if self.current_item:
db.query(
"""
INSERT INTO asrun (id_channel, id_item, start)
VALUES (%s, %s, %s)
""",
[self.id_channel, item.id,
time.time()],
)
self.last_run = db.lastid()
db.commit()
else:
self.last_run = False
for plugin in self.plugins:
try:
plugin.on_change()
except Exception:
log_traceback("Plugin on-change failed")
def set_progress(self, progress, message="In progress"):
db = DB()
progress = round(progress, 2)
db.query(
"""
UPDATE jobs SET
status=1,
progress=%s,
message=%s
WHERE id=%s
""",
[progress, message, self.id],
)
db.commit()
messaging.send(
"job_progress",
id=self.id,
id_asset=self.id_asset,
id_action=self.id_action,
status=JobState.IN_PROGRESS,
progress=progress,
message=message,
)
def s(*args):
print()
db = DB()
if len(args) >= 2:
try:
services = tuple([int(i.strip()) for i in args[1:]])
except ValueError:
critical_error("Wrong service ID")
if args[0] == "start":
db.query("UPDATE services SET state=2 WHERE id IN %s AND state=0",
[services])
db.commit()
elif args[0] == "stop":
db.query("UPDATE services SET state=3 WHERE id IN %s AND state=1",
[services])
db.commit()
elif args[0] == "kill":
db.query(
"UPDATE services SET state=4 WHERE id IN %s AND state IN (1,3)",
[services],
)
db.commit()
elif args[0] == "auto":
db.query("UPDATE services SET autostart=TRUE WHERE id IN %s",
[services])
db.commit()
elif args[0] == "noauto":
db.query("UPDATE services SET autostart=FALSE WHERE id IN %s",
[services])
db.commit()
else:
critical_error("Unsupported command: {}".format(args[0]))
time.sleep(1)
show_services(db)
def update_host_info():
hostname = config["host"]
mp2id = {}
for id_storage, storage in storages.items():
mp2id[storage.local_path] = id_storage
p = Promexp(provider_settings={"casparcg": None})
p.metrics.add("runtime_seconds", time.time() - NEBULA_START_TIME)
p.collect()
for metric in list(p.metrics.data.keys()):
mname, tags = metric
if mname in [
"storage_bytes_total", "storage_bytes_free", "storage_usage"
]:
id_storage = mp2id.get(tags["mountpoint"])
if id_storage is None:
continue
value = p.metrics.data[metric]
del p.metrics.data[metric]
p.metrics.add(
f"shared_{mname}",
value,
id=id_storage,
title=storages[id_storage].title,
)
status = {"metrics": p.metrics.dump()}
db = DB()
db.query(
"UPDATE hosts SET last_seen=%s, status=%s WHERE hostname=%s",
[time.time(), json.dumps(status), hostname],
)
db.commit()
def soft_stop(self):
logging.info("Soft stop requested")
db = DB()
db.query("UPDATE services SET state=3 WHERE id=%s", [self.id_service])
db.commit()
def main(self):
db = DB()
db.query(
"""
SELECT
id,
title,
autostart,
state,
last_seen
FROM services
WHERE host=%s
""",
[config["host"]],
)
#
# Start / stop service
#
for id, title, autostart, state, last_seen in db.fetchall():
messaging.send(
"service_state",
id=id,
state=state,
autostart=autostart,
last_seen=last_seen,
last_seen_before=max(0, int(time.time() - last_seen)),
)
if state == ServiceState.STARTING: # Start service
if id not in self.services.keys():
self.start_service(id, title, db=db)
elif state == ServiceState.KILL: # Kill service
if id in self.services.keys():
self.kill_service(self.services[id][0].pid)
#
# Real service state
#
service_list = [i for i in self.services.keys()]
for id_service in service_list:
proc, title = self.services[id_service]
if proc.poll() is None:
continue
del self.services[id_service]
logging.warning(f"Service ID {id_service} ({title}) terminated")
db.query("UPDATE services SET state=0 WHERE id = %s", [id_service])
db.commit()
#
# Autostart
#
db.query(
"""
SELECT id, title, state, autostart
FROM services
WHERE host=%s AND state=0 AND autostart=true
""",
[config["host"]],
)
for id, title, state, autostart in db.fetchall():
if id not in self.services.keys():
logging.debug(f"AutoStarting service ID {id} ({title})")
self.start_service(id, title)
def api_system(**kwargs):
"""
Returns system status and controls services
Arguments:
request list of information to show.
defaults to everything ["services", "hosts"]
stop stop service by its ID
start start service by its ID
kill kill service by its ID
set_autostart toggle service autostart param (True/False)
"""
user = kwargs.get("user", anonymous)
message = ""
if not user:
return NebulaResponse(401)
db = DB()
request = kwargs.get("request", ["services", "hosts"])
if "stop" in kwargs:
id_service = kwargs["stop"]
if not type(id_service) == int:
return NebulaResponse(400, "Invalid ID service to stop")
if not user.has_right("service_control", id_service):
return NebulaResponse(
403, "You are not allowed to control this service")
db.query("UPDATE services SET state=3 WHERE id=%s AND state = 1",
[id_service])
db.commit()
logging.info(f"{user} requested service ID {id_service} "
f"({config['services'][id_service]['title']}) stop")
message = "Service is stopping"
if "start" in kwargs:
id_service = kwargs["start"]
if not type(id_service) == int:
return NebulaResponse(400, "Invalid ID service to start")
if not user.has_right("service_control", id_service):
return NebulaResponse(
403, "You are not allowed to control this service")
db.query("UPDATE services SET state=2 WHERE id=%s AND state = 0",
[id_service])
db.commit()
logging.info(f"{user} requested service ID {id_service} "
f"({config['services'][id_service]['title']}) start")
message = "Service is starting"
if "kill" in kwargs:
id_service = kwargs["kill"]
if not type(id_service) == int:
return NebulaResponse(400, "Invalid ID service to kill")
if not user.has_right("service_control", id_service):
return NebulaResponse(
403, "You are not allowed to control this service")
db.query("UPDATE services SET state=4 WHERE id=%s AND state = 3",
[id_service])
db.commit()
logging.info(f"{user} requested service ID {id_service} "
f"({config['services'][id_service]['title']}) kill")
message = "Attempting to kill the service"
if "autostart" in kwargs:
id_service = kwargs["autostart"]
if not type(id_service) == int:
return NebulaResponse(400, "Invalid ID service to set autostart")
if not user.has_right("service_control", id_service):
return NebulaResponse(
403, "You are not allowed to control this service")
db.query("UPDATE services SET autostart=NOT autostart WHERE id=%s",
[id_service])
logging.info(f"{user} requested service ID {id_service} "
f"({config['services'][id_service]['title']}) autostart")
db.commit()
message = "Service auto-start updated"
result = {}
if "services" in request:
services = []
db.query("""
SELECT id, service_type, host, title, autostart, state, last_seen
FROM services ORDER BY id ASC
""")
for id, service_type, host, title, autostart, state, last_seen in db.fetchall(
):
service = {
"id": id,
"type": service_type,
"host": host,
"title": title,
"autostart": autostart,
"state": state,
"last_seen": last_seen,
"last_seen_before": time.time() - last_seen,
}
services.append(service)
result["services"] = services
if "hosts" in request:
hosts = []
db.query(
"SELECT hostname, last_seen, status FROM hosts ORDER BY hostname")
for hostname, last_seen, status in db.fetchall():
host = status
host["hostname"] = hostname
host["last_seen"] = last_seen
hosts.append(host)
result["hosts"] = hosts
return NebulaResponse(200, data=result, message=message)
def run(*args):
id_service = args[0]
if id_service == "hub":
import hub
try:
hub_instance = hub.CherryAdmin(**hub.hub_config)
except Exception:
log_traceback()
critical_error("Unhandled exception in Hub")
return
try:
id_service = int(id_service)
except ValueError:
critical_error("Service ID must be integer")
db = DB()
db.query(
"""
SELECT service_type, title, host, loop_delay, settings
FROM services WHERE id=%s
""",
[id_service],
)
try:
agent, title, host, loop_delay, settings = db.fetchall()[0]
except IndexError:
critical_error(
f"Unable to start service {id_service}. No such service")
config["user"] = logging.user = title
if host != config["host"]:
critical_error("This service should not run here.")
if settings:
try:
settings = xml(settings)
except Exception:
log_traceback()
logging.error("Malformed settings XML:\n", settings)
db.query("UPDATE services SET autostart=0 WHERE id=%s",
[id_service])
db.commit()
critical_error("Unable to start service")
_module = __import__("services." + agent, globals(), locals(), ["Service"])
Service = _module.Service
service = Service(id_service, settings)
while True:
try:
service.on_main()
last_run = time.time()
while True:
time.sleep(min(loop_delay, 2))
service.heartbeat()
if time.time() - last_run >= loop_delay:
break
except (KeyboardInterrupt):
logging.warning("Keyboard interrupt")
break
except (SystemExit):
break
except Exception:
log_traceback()
time.sleep(2)
sys.exit(1)
try:
if sys.argv[1] == "once":
break
except IndexError:
pass