def post(self, model):
data, result = request.get_json(force=True), defaultdict(list)
if not isinstance(data, list):
data = [data]
for instance in data:
if "name" not in instance:
result["failure"].append((instance, "Name is missing"))
continue
try:
object_data = app.objectify(model, instance)
object_data["update_pools"] = instance.get(
"update_pools", True)
instance = db.factory(model, **object_data)
result["success"].append(instance.name)
except Exception:
result["failure"].append((instance, format_exc()))
return result
def update(self, type, **kwargs):
try:
must_be_new = kwargs.get("id") == ""
for arg in ("name", "scoped_name"):
if arg in kwargs:
kwargs[arg] = kwargs[arg].strip()
kwargs["last_modified"] = self.get_time()
kwargs["creator"] = kwargs["user"] = getattr(current_user, "name", "")
instance = db.factory(type, must_be_new=must_be_new, **kwargs)
if kwargs.get("original"):
db.fetch(type, id=kwargs["original"]).duplicate(clone=instance)
db.session.flush()
return instance.serialized
except Exception as exc:
db.session.rollback()
if isinstance(exc, IntegrityError):
return {"alert": f"There is already a {type} with the same parameters."}
return {"alert": str(exc)}
def send_data(session, file_descriptor):
session_object = db.factory(
"session",
commit=True,
name=session,
timestamp=str(datetime.now()),
**vs.ssh_sessions[session],
)
while True:
try:
self.socketio.sleep(0.1)
output = read(file_descriptor, 1024).decode()
session_object.content += output
self.socketio.emit("output",
output,
namespace="/terminal",
room=session)
db.session.commit()
except OSError:
break
def authenticate_user(self, **kwargs):
name, password = kwargs["name"], kwargs["password"]
if not name or not password:
return False
user = db.fetch("user", allow_none=True, name=name)
method = kwargs.get("authentication_method",
getattr(user, "authentication", None))
if not method or not self.settings["authentication"].get(method):
return False
elif method == "database":
hash = self.settings["security"]["hash_user_passwords"]
verify = argon2.verify if hash else str.__eq__
user_password = self.get_password(user.password)
return user if user and verify(password, user_password) else False
else:
response = getattr(self, f"{method}_authentication")(user, name,
password)
if not response:
return False
elif not user:
user = db.factory("user", authentication=method, **response)
db.session.commit()
return user
def tracking_bfs(self, run, payload):
number_of_runs = defaultdict(int)
start = db.fetch("service", scoped_name="Start")
end = db.fetch("service", scoped_name="End")
services = [db.fetch("service", id=id) for id in run.start_services]
visited, success, targets = set(), False, defaultdict(set)
restart_run = run.restart_run
for service in services:
targets[service.name] |= {device.name for device in run.devices}
while services:
if run.stop:
return {"payload": payload, "success": False}
service = services.pop()
if number_of_runs[service.name] >= service.maximum_runs or any(
node not in visited
for node, _ in service.adjacent_services(
self, "source", "prerequisite")):
continue
number_of_runs[service.name] += 1
visited.add(service)
if service in (start, end):
results = {
"summary": {
"success": {device.name
for device in run.devices},
"failure": [],
},
"success": True,
}
else:
kwargs = {
"devices": [
db.fetch("device", name=name).id
for name in targets[service.name]
],
"service":
run.placeholder.id
if service.scoped_name == "Placeholder" else service.id,
"workflow":
self.id,
"restart_run":
restart_run,
"parent":
run,
"parent_runtime":
run.parent_runtime,
}
if run.parent_device_id:
kwargs["parent_device"] = run.parent_device_id
service_run = db.factory("run", **kwargs)
results = service_run.run(payload)
if service.run_method in ("once",
"per_service_with_service_targets"):
edge_type = "success" if results["success"] else "failure"
for successor, edge in service.adjacent_services(
self, "destination", edge_type):
targets[successor.name] |= targets[service.name]
services.append(successor)
run.edge_state[edge.id] += len(targets[service.name])
else:
summary = results.get("summary", {})
for edge_type in ("success", "failure"):
for successor, edge in service.adjacent_services(
self,
"destination",
edge_type,
):
if not summary[edge_type]:
continue
targets[successor.name] |= set(summary[edge_type])
services.append(successor)
run.edge_state[edge.id] += len(summary[edge_type])
success_devices = targets[end.name]
failure_devices = targets[start.name] - success_devices
success = not failure_devices
summary = {
"success": list(success_devices),
"failure": list(failure_devices),
}
run.run_state["progress"]["device"]["success"] = len(success_devices)
run.run_state["progress"]["device"]["failure"] = len(failure_devices)
run.run_state["summary"] = summary
db.session.refresh(run)
run.restart_run = restart_run
return {"payload": payload, "success": success}
def migration_import(self, folder="migrations", **kwargs):
status, models = "Import successful.", kwargs["import_export_types"]
if kwargs.get("empty_database_before_import", False):
db.delete_all(*models)
workflow_edges, workflow_services, superworkflows = [], {}, {}
folder_path = self.path / "files" / folder / kwargs["name"]
for model in models:
path = folder_path / f"{model}.yaml"
if not path.exists():
continue
with open(path, "r") as migration_file:
instances = yaml.load(migration_file)
if model == "workflow_edge":
workflow_edges = deepcopy(instances)
continue
for instance in instances:
instance_type = (
instance.pop("type") if model == "service" else model
)
if (
instance_type in ("service", "workflow")
and "superworkflow" in instance
):
superworkflows[instance["name"]] = instance.pop("superworkflow")
if instance_type == "workflow":
workflow_services[instance["name"]] = instance.pop("services")
try:
instance = self.objectify(instance_type, instance)
db.factory(
instance_type, **{"dont_update_pools": True, **instance}
)
db.session.commit()
except Exception:
info(f"{str(instance)} could not be imported:\n{format_exc()}")
status = "Partial import (see logs)."
try:
for name, services in workflow_services.items():
workflow = db.fetch("workflow", name=name)
workflow.services = [
db.fetch("service", name=service_name) for service_name in services
]
db.session.commit()
for name, superworkflow in superworkflows.items():
service = db.fetch("service", name=name)
service.superworkflow = db.fetch("workflow", name=superworkflow)
db.session.commit()
for edge in workflow_edges:
for property in ("source", "destination", "workflow"):
edge[property] = db.fetch("service", name=edge[property]).id
self.update(edge.pop("type"), **edge)
db.session.commit()
for service in db.fetch_all("service"):
service.update()
if not kwargs.get("skip_pool_update"):
for pool in db.fetch_all("pool"):
pool.compute_pool()
self.log("info", status)
except Exception:
info("\n".join(format_exc().splitlines()))
status = "Partial import (see logs)."
return status
def post(self, cls):
data = request.get_json(force=True)
object_data = app.objectify(cls, data)
result = db.factory(cls, **object_data).serialized
db.session.commit()
return result
def duplicate(self, **kwargs):
properties = {
k: v for (k, v) in self.get_properties().items() if k not in ("id", "name")
}
instance = db.factory(self.type, **{**properties, **kwargs})
return instance
def create_admin_user(self):
admin = db.factory("user", name="admin", is_admin=True)
if not admin.password:
admin.update(password="******")
def job(self, run, payload, device=None):
number_of_runs = defaultdict(int)
start = db.fetch("service", scoped_name="Start")
end = db.fetch("service", scoped_name="End")
services = [db.fetch("service", id=id) for id in run.start_services]
visited, targets, restart_run = set(), defaultdict(
set), run.restart_run
tracking_bfs = run.run_method == "per_service_with_workflow_targets"
start_targets = [device] if device else run.target_devices
for service in services:
targets[service.name] |= {device.name for device in start_targets}
while services:
if run.stop:
return {
"payload": payload,
"success": False,
"result": "Stopped"
}
service = services.pop()
if number_of_runs[service.name] >= service.maximum_runs or any(
node not in visited for node, _ in service.neighbors(
self, "source", "prerequisite")):
continue
number_of_runs[service.name] += 1
visited.add(service)
if service in (start, end) or service.skip.get(self.name, False):
success = service.skip_value == "success"
results = {"result": "skipped", "success": success}
if tracking_bfs or device:
results["summary"] = {
"success": targets[service.name],
"failure": [],
}
else:
kwargs = {
"service":
run.placeholder.id
if service.scoped_name == "Placeholder" else service.id,
"workflow":
self.id,
"restart_run":
restart_run,
"parent":
run,
"parent_runtime":
run.parent_runtime,
}
if tracking_bfs or device:
kwargs["target_devices"] = [
db.fetch("device", name=name).id
for name in targets[service.name]
]
if run.parent_device_id:
kwargs["parent_device"] = run.parent_device_id
service_run = db.factory("run", commit=True, **kwargs)
results = service_run.run(payload)
if not results:
continue
status = "success" if results["success"] else "failure"
summary = results.get("summary", {})
if not tracking_bfs and not device:
run.write_state(f"progress/service/{status}", 1, "increment")
for edge_type in ("success", "failure"):
if not tracking_bfs and edge_type != status:
continue
if tracking_bfs and not summary[edge_type]:
continue
for successor, edge in service.neighbors(
self, "destination", edge_type):
if tracking_bfs or device:
targets[successor.name] |= set(summary[edge_type])
services.append(successor)
if tracking_bfs or device:
run.write_state(f"edges/{edge.id}",
len(summary[edge_type]), "increment")
else:
run.write_state(f"edges/{edge.id}", "DONE")
if tracking_bfs or device:
failed = list(targets[start.name] - targets[end.name])
summary = {"success": list(targets[end.name]), "failure": failed}
results = {
"payload": payload,
"success": not failed,
"summary": summary
}
else:
results = {"payload": payload, "success": end in visited}
db.session.refresh(run)
run.restart_run = restart_run
return results
def migration_import(self, folder="migrations", **kwargs):
status, models = "Import successful.", kwargs["import_export_types"]
empty_database = kwargs.get("empty_database_before_import", False)
if empty_database:
db.delete_all(*models)
relations = defaultdict(lambda: defaultdict(dict))
for model in models:
path = self.path / "files" / folder / kwargs[
"name"] / f"{model}.yaml"
if not path.exists():
continue
with open(path, "r") as migration_file:
instances = yaml.load(migration_file)
for instance in instances:
instance_type, relation_dict = instance.pop("type",
model), {}
for related_model, relation in relationships[
instance_type].items():
relation_dict[related_model] = instance.pop(
related_model, [])
for property, value in instance.items():
if property in db.private_properties_list:
instance[property] = self.get_password(value)
try:
instance = db.factory(
instance_type,
migration_import=True,
no_fetch=empty_database,
update_pools=kwargs.get("update_pools", False),
import_mechanism=True,
**instance,
)
relations[instance_type][instance.name] = relation_dict
except Exception:
info(
f"{str(instance)} could not be imported:\n{format_exc()}"
)
status = "Partial import (see logs)."
db.session.commit()
for model, instances in relations.items():
for instance_name, related_models in instances.items():
for property, value in related_models.items():
if not value:
continue
relation = relationships[model][property]
if relation["list"]:
related_instances = (db.fetch(relation["model"],
name=name,
allow_none=True)
for name in value)
value = list(filter(None, related_instances))
else:
value = db.fetch(relation["model"],
name=value,
allow_none=True)
try:
setattr(db.fetch(model, name=instance_name), property,
value)
except Exception:
info("\n".join(format_exc().splitlines()))
status = "Partial import (see logs)."
db.session.commit()
if not kwargs.get("skip_model_update"):
for model in ("access", "service"):
for instance in db.fetch_all(model):
instance.update()
if not kwargs.get("skip_pool_update"):
for pool in db.fetch_all("pool"):
pool.compute_pool()
self.log("info", status)
return status
def create_admin_user(self) -> None:
admin = db.factory("user", **{"name": "admin", "group": "Admin"})
if not admin.password:
admin.update(password="******")
def create_admin_user(self) -> None:
admin = db.factory("user", name="admin")
if not admin.password:
admin.update(password="******")
