def query_opennms(self, **kwargs: str) -> None:
login, password = self.opennms_login, kwargs["password"]
Session.commit()
json_devices = http_get(
self.opennms_devices,
headers={"Accept": "application/json"},
auth=(login, password),
).json()["node"]
devices = {
device["id"]: {
"name": device.get("label", device["id"]),
"description": device["assetRecord"].get("description", ""),
"location": device["assetRecord"].get("building", ""),
"vendor": device["assetRecord"].get("manufacturer", ""),
"model": device["assetRecord"].get("modelNumber", ""),
"operating_system": device.get("operatingSystem", ""),
"os_version": device["assetRecord"].get("sysDescription", ""),
"longitude": device["assetRecord"].get("longitude", 0.0),
"latitude": device["assetRecord"].get("latitude", 0.0),
}
for device in json_devices
}
for device in list(devices):
link = http_get(
f"{self.opennms_rest_api}/nodes/{device}/ipinterfaces",
headers={"Accept": "application/json"},
auth=(login, password),
).json()
for interface in link["ipInterface"]:
if interface["snmpPrimary"] == "P":
devices[device]["ip_address"] = interface["ipAddress"]
factory("Device", **devices[device])
def import_jobs(self, **kwargs: Any) -> None:
jobs = kwargs["jobs_to_import"]
path = self.path / "projects" / "exported_jobs"
for file in scandir(path / "services"):
if file.name == ".gitkeep" or file.name not in jobs:
continue
with open(file.path, "r") as instance_file:
instance = yaml.load(instance_file)
model = instance.pop("type")
factory(model, **self.objectify(model, instance))
Session.commit()
for workflow in listdir(path / "workflows"):
if workflow == ".gitkeep" or workflow not in jobs:
continue
workflow_name = workflow.split(".")[0]
with open_tar(path / "workflows" / workflow) as tar_file:
tar_file.extractall(path=path / "workflows")
for instance_type in ("jobs", "workflow", "edges"):
path_job = path / "workflows" / workflow_name / instance_type
for file in scandir(path_job):
with open(path_job / file.name, "r") as instance_file:
instance = yaml.load(instance_file)
model = instance.pop("type")
factory(model, **self.objectify(model, instance))
Session.commit()
rmtree(path / "workflows" / workflow_name)
def migration_import(self, folder="migrations", **kwargs):
status, models = "Import successful.", kwargs["import_export_types"]
skip_update_pools_after_import = kwargs.get(
"skip_update_pools_after_import", False
)
if kwargs.get("empty_database_before_import", False):
for model in models:
delete_all(model)
Session.commit()
workflow_edges, workflow_services = [], {}
folder_path = self.path / "files" / folder / kwargs["name"]
for model in models:
path = folder_path / f"{model}.yaml"
if not path.exists():
continue
with open(path, "r") as migration_file:
instances = yaml.load(migration_file)
if model == "workflow_edge":
workflow_edges = deepcopy(instances)
continue
for instance in instances:
instance_type = (
instance.pop("type") if model == "service" else model
)
if instance_type == "workflow":
workflow_services[instance["name"]] = instance.pop("services")
try:
instance = self.objectify(instance_type, instance)
factory(
instance_type, **{"dont_update_pools": True, **instance}
)
Session.commit()
except Exception:
info(
f"{str(instance)} could not be imported :"
f"{chr(10).join(format_exc().splitlines())}"
)
status = "Partial import (see logs)."
try:
for name, services in workflow_services.items():
workflow = fetch("workflow", name=name)
workflow.services = [
fetch("service", name=service_name) for service_name in services
]
Session.commit()
for edge in workflow_edges:
for property in ("source", "destination", "workflow"):
edge[property] = fetch("service", name=edge[property]).id
factory("workflow_edge", **edge)
Session.commit()
for service in fetch_all("service"):
service.set_name()
if not skip_update_pools_after_import:
for pool in fetch_all("pool"):
pool.compute_pool()
self.log("info", status)
except Exception:
info(chr(10).join(format_exc().splitlines()))
status = "Partial import (see logs)."
return status
def topology_import(self, file):
book = open_workbook(file_contents=file.read())
status = "Topology successfully imported."
for obj_type in ("device", "link"):
try:
sheet = book.sheet_by_name(obj_type)
except XLRDError:
continue
properties = sheet.row_values(0)
for row_index in range(1, sheet.nrows):
values = {"dont_update_pools": True}
for index, property in enumerate(properties):
if not property:
continue
func = field_conversion[property_types.get(
property, "str")]
values[property] = func(sheet.row_values(row_index)[index])
try:
factory(obj_type, **values).serialized
except Exception as exc:
info(f"{str(values)} could not be imported ({str(exc)})")
status = "Partial import (see logs)."
Session.commit()
for pool in fetch_all("pool"):
pool.compute_pool()
self.log("info", status)
return status
def duplicate_workflow(self, workflow_id, **kwargs):
parent_workflow = fetch("workflow", id=workflow_id)
new_workflow = factory("workflow", **kwargs)
Session.commit()
for service in parent_workflow.services:
new_workflow.services.append(service)
service.positions[new_workflow.name] = service.positions[
parent_workflow.name]
Session.commit()
for edge in parent_workflow.edges:
subtype, src, destination = edge.subtype, edge.source, edge.destination
new_workflow.edges.append(
factory(
"workflow_edge",
**{
"name": (f"{new_workflow.id}-{subtype}:"
f"{src.id}->{destination.id}"),
"workflow":
new_workflow.id,
"subtype":
subtype,
"source":
src.id,
"destination":
destination.id,
},
))
return new_workflow.serialized
def duplicate_workflow(self, workflow_id: int, **kwargs: Any) -> dict:
parent_workflow = fetch("Workflow", id=workflow_id)
new_workflow = factory("Workflow", **kwargs)
Session.commit()
for job in parent_workflow.jobs:
new_workflow.jobs.append(job)
job.positions[new_workflow.name] = job.positions[
parent_workflow.name]
Session.commit()
for edge in parent_workflow.edges:
subtype, src, destination = edge.subtype, edge.source, edge.destination
new_workflow.edges.append(
factory(
"WorkflowEdge",
**{
"name": (f"{new_workflow.id}-{subtype}:"
f"{src.id}->{destination.id}"),
"workflow":
new_workflow.id,
"subtype":
subtype,
"source":
src.id,
"destination":
destination.id,
},
))
return new_workflow.serialized
def create_result(self,
results: dict,
device: Optional["Device"] = None) -> None:
self.success = results["success"]
result_kw = {"run": self, "result": results}
if device:
result_kw["device"] = device.id
factory("Result", **result_kw)
def configure_server_id(self):
factory(
"server",
**{
"name": str(getnode()),
"description": "Localhost",
"ip_address": "0.0.0.0",
"status": "Up",
},
)
def log(self, severity, content):
factory(
"changelog",
**{
"severity": severity,
"content": content,
"user": getattr(current_user, "name", "admin"),
},
)
self.log_severity[severity](content)
def cluster_monitoring(self, run: "Run", payload: dict) -> dict:
protocol = controller.cluster_scan_protocol
for instance in fetch_all("Instance"):
factory(
"Instance",
**get(
f"{protocol}://{instance.ip_address}/rest/is_alive",
timeout=controller.cluster_scan_timeout,
).json(),
)
return {"success": True}
def cluster_monitoring(self, run, payload):
protocol = app.settings["cluster"]["scan_protocol"]
for instance in fetch_all("instance"):
factory(
"instance",
**get(
f"{protocol}://{instance.ip_address}/rest/is_alive",
timeout=app.settings["cluster"]["scan_timeout"],
).json(),
)
return {"success": True}
def scan_cluster(self, **kwargs: Union[float, str]) -> None:
for ip_address in IPv4Network(self.cluster_scan_subnet):
try:
server = http_get(
f"{self.cluster_scan_protocol}://{ip_address}/rest/is_alive",
timeout=self.cluster_scan_timeout,
).json()
if self.cluster_id != server.pop("cluster_id"):
continue
factory("Server", **{**server, **{"ip_address": str(ip_address)}})
except ConnectionError:
continue
def scan_cluster(self, **kwargs):
protocol = self.config["cluster"]["scan_protocol"]
for ip_address in IPv4Network(self.config["cluster"]["scan_subnet"]):
try:
server = http_get(
f"{protocol}://{ip_address}/rest/is_alive",
timeout=self.config["cluster"]["scan_timeout"],
).json()
if self.config["cluster"]["id"] != server.pop("cluster_id"):
continue
factory("server", **{**server, **{"ip_address": str(ip_address)}})
except ConnectionError:
continue
def create_result(self, results, device=None):
self.success = results["success"]
result_kw = {
"run": self,
"result": results,
"service": self.service_id
}
if self.service.type == "workflow":
result_kw["workflow"] = self.service_id
elif self.workflow_id:
result_kw["workflow"] = self.workflow_id
if device:
result_kw["device"] = device.id
factory("result", **result_kw)
def create_result(self, results, device=None):
self.success = results["success"]
result_kw = {
"run": self,
"result": results,
"service": self.service_id,
"parent_runtime": self.parent_runtime,
}
if self.workflow_id:
result_kw["workflow"] = self.workflow_id
if self.parent_device_id:
result_kw["parent_device"] = self.parent_device_id
if device:
result_kw["device"] = device.id
factory("result", **result_kw)
def device_iteration(self, payload, device):
derived_devices = self.compute_devices_from_query(
self.iteration_devices, self.iteration_devices_property,
**locals())
derived_run = factory(
"run",
**{
"service":
self.service.id,
"devices":
[derived_device.id for derived_device in derived_devices],
"workflow":
self.workflow.id,
"parent_device":
device.id,
"restart_run":
self.restart_run,
"parent":
self,
"parent_runtime":
self.parent_runtime,
},
)
derived_run.properties = self.properties
return derived_run.run(payload)["success"]
def duplicate(self, workflow=None, clone=None):
if not clone:
clone = super().duplicate(workflow)
clone_services = {}
Session.commit()
for service in self.services:
if service.shared:
service_clone = service
if service not in clone.services:
clone.services.append(service)
else:
service_clone = service.duplicate(clone)
service_clone.positions[clone.name] = service.positions.get(
self.name, (0, 0))
clone_services[service.id] = service_clone
Session.commit()
for edge in self.edges:
clone.edges.append(
factory(
"workflow_edge",
**{
"workflow": clone.id,
"subtype": edge.subtype,
"source": clone_services[edge.source.id].id,
"destination": clone_services[edge.destination.id].id,
},
))
Session.commit()
return clone
def handoffssh(self, id, **kwargs):
device = fetch("device", id=id)
credentials = (
(device.username,
device.password) if kwargs["credentials"] == "device" else
(current_user.name,
current_user.password) if kwargs["credentials"] == "user" else
(kwargs["username"], kwargs["password"]))
uuid, port = str(uuid4()), self.get_ssh_port()
session = factory(
"session",
name=uuid,
user=current_user.name,
timestamp=self.get_time(),
device=device.id,
)
Session.commit()
try:
ssh_connection = SshConnection(device.ip_address, *credentials,
session.id, uuid, port)
Thread(
target=ssh_connection.start_session,
args=(session.id, uuid, port),
).start()
return {
"port": port,
"username": uuid,
"device_name": device.name,
"device_ip": device.ip_address,
}
except Exception as exc:
return {"error": exc.args}
def run(service, **kwargs):
run_kwargs = {
key: kwargs.pop(key)
for key in (
"creator",
"restart_path",
"start_services",
"runtime",
"task",
"devices",
"pools",
) if kwargs.get(key)
}
restart_run = fetch(
"run",
allow_none=True,
runtime=kwargs.get("restart_runtime"),
)
if restart_run:
run_kwargs["restart_run"] = restart_run
initial_payload = fetch("service", id=service).initial_payload
run = factory("run", service=service, **run_kwargs)
run.properties = kwargs
payload = {**initial_payload, **kwargs}
return run.run(payload)
def device_iteration(self, payload, device):
derived_devices = self.compute_devices_from_query(
self.service.iteration_devices,
self.service.iteration_devices_property,
**locals(),
)
derived_run = factory(
"run",
**{
"service":
self.service.id,
"devices":
[derived_device.id for derived_device in derived_devices],
"workflow":
self.workflow.id,
"parent_device":
device.id,
"restart_run":
self.restart_run,
"parent":
self,
"parent_runtime":
self.parent_runtime,
},
)
derived_run.properties = self.properties
success = derived_run.run(payload)["success"]
key = "success" if success else "failure"
self.run_state["summary"][key].append(device.name)
return success
def update(self, instance_type, **kwargs):
try:
must_be_new = kwargs.get("id") == ""
for arg in ("name", "scoped_name"):
if arg in kwargs:
kwargs[arg] = kwargs[arg].strip()
kwargs["last_modified"] = self.get_time()
kwargs["creator"] = kwargs["user"] = getattr(
current_user, "name", "admin")
instance = factory(instance_type,
must_be_new=must_be_new,
**kwargs)
if kwargs.get("original"):
fetch(instance_type,
id=kwargs["original"]).duplicate(clone=instance)
Session.flush()
return instance.serialized
except Exception as exc:
Session.rollback()
if isinstance(exc, IntegrityError):
return {
"alert":
(f"There already is a {instance_type} with the same name")
}
return {"alert": str(exc)}
def post(self):
data = request.get_json(force=True)
factory(
"pool",
**{
"name": data["name"],
"devices": [
fetch("device", name=name).id for name in data.get("devices", "")
],
"links": [
fetch("link", name=name).id for name in data.get("links", "")
],
"never_update": True,
},
)
Session.commit()
return data
def duplicate(self, **kwargs):
properties = {
k: v
for (k, v) in self.get_properties().items()
if k not in ("id", "name")
}
instance = factory(self.type, **{**properties, **kwargs})
return instance
def post(self, cls: str) -> dict:
try:
data = request.get_json(force=True)
object_data = controller.objectify(cls.capitalize(), data)
result = factory(cls, **object_data).serialized
Session.commit()
return result
except Exception as exc:
return abort(500, message=f"Update failed ({exc})")
def post(self, cls):
try:
data = request.get_json(force=True)
object_data = app.objectify(cls, data)
result = factory(cls, **object_data).serialized
Session.commit()
return result
except Exception as exc:
return abort(500, message=f"Update failed ({exc})")
def query_netbox(self, **kwargs: str) -> None:
nb = netbox_api(kwargs["netbox_address"], token=kwargs["netbox_token"])
for device in nb.dcim.devices.all():
device_ip = device.primary_ip4 or device.primary_ip6
factory(
"Device",
**{
"name": device.name,
"ip_address": str(device_ip).split("/")[0],
"subtype": str(device.device_role),
"model": str(device.device_type),
"location": str(device.site),
"vendor": str(device.device_type.manufacturer),
"operating_system": str(device.platform),
"longitude": str(nb.dcim.sites.get(name=device.site).longitude),
"latitude": str(nb.dcim.sites.get(name=device.site).latitude),
},
)
def run_job(job: int, **kwargs: Any) -> dict:
run_kwargs = {
key: kwargs.pop(key)
for key in ("creator", "runtime", "task", "restart_runtime")
if kwargs.get(key)
}
run = factory("Run", job=job, **run_kwargs)
run.properties = kwargs
return run.run(kwargs.get("payload"))
def update(self, cls: str, **kwargs: Any) -> dict:
try:
instance = factory(cls, **kwargs)
Session.commit()
return instance.serialized
except JSONDecodeError:
return {"error": "Invalid JSON syntax (JSON field)"}
except IntegrityError:
return {"error": "An object with the same name already exists"}
def migration_import(self, **kwargs: Any) -> str:
status, types = "Import successful.", kwargs["import_export_types"]
if kwargs.get("empty_database_before_import", False):
for type in types:
delete_all(type)
Session.commit()
workflow_edges: list = []
for cls in types:
path = (self.path / "projects" / "migrations" / kwargs["name"] /
f"{cls}.yaml")
with open(path, "r") as migration_file:
objects = yaml.load(migration_file)
if cls == "Workflow":
workflow_jobs = {
workflow["name"]: workflow.pop("jobs")
for workflow in objects
}
if cls == "WorkflowEdge":
workflow_edges = deepcopy(objects)
if cls == "Service":
objects.sort(key=lambda s: s["type"] == "IterationService")
for obj in objects:
obj_cls = obj.pop("type") if cls == "Service" else cls
obj = self.objectify(obj_cls, obj)
try:
factory(obj_cls, **obj)
Session.commit()
except Exception as e:
info(f"{str(obj)} could not be imported ({str(e)})")
if cls in ("Service", "Workflow"):
Session.commit()
status = "Partial import (see logs)."
if cls not in ("Service", "Workflow"):
Session.commit()
for name, jobs in workflow_jobs.items():
fetch("Workflow",
name=name).jobs = [fetch("Job", name=name) for name in jobs]
Session.commit()
for edge in workflow_edges:
for property in ("source", "destination", "workflow"):
edge[property] = fetch("Job", name=edge[property]).id
factory("WorkflowEdge", **edge)
Session.commit()
return status
def migration_import(self, folder="migrations", **kwargs):
status, models = "Import successful.", kwargs["import_export_types"]
if kwargs.get("empty_database_before_import", False):
for model in models:
delete_all(model)
Session.commit()
workflow_edges, workflow_services = [], {}
folder_path = self.path / "projects" / folder / kwargs["name"]
for model in models:
path = folder_path / f"{model}.yaml"
if not path.exists():
continue
with open(path, "r") as migration_file:
instances = yaml.load(migration_file)
if model == "workflow_edge":
workflow_edges = deepcopy(instances)
continue
for instance in instances:
instance_type = (instance.pop("type")
if model == "service" else model)
if instance_type == "workflow":
workflow_services[instance["name"]] = instance.pop(
"services")
instance = self.objectify(instance_type, instance)
try:
factory(instance_type, **instance)
Session.commit()
except Exception as e:
info(
f"{str(instance)} could not be imported ({str(e)})"
)
status = "Partial import (see logs)."
for name, services in workflow_services.items():
fetch("workflow", name=name).services = [
fetch("service", name=service_name)
for service_name in services
]
Session.commit()
for edge in workflow_edges:
for property in ("source", "destination", "workflow"):
edge[property] = fetch("service", name=edge[property]).id
factory("workflow_edge", **edge)
Session.commit()
return status
