def migration_import(self, folder="migrations", **kwargs):
status, models = "Import successful.", kwargs["import_export_types"]
skip_update_pools_after_import = kwargs.get(
"skip_update_pools_after_import", False
)
if kwargs.get("empty_database_before_import", False):
for model in models:
delete_all(model)
Session.commit()
workflow_edges, workflow_services = [], {}
folder_path = self.path / "files" / folder / kwargs["name"]
for model in models:
path = folder_path / f"{model}.yaml"
if not path.exists():
continue
with open(path, "r") as migration_file:
instances = yaml.load(migration_file)
if model == "workflow_edge":
workflow_edges = deepcopy(instances)
continue
for instance in instances:
instance_type = (
instance.pop("type") if model == "service" else model
)
if instance_type == "workflow":
workflow_services[instance["name"]] = instance.pop("services")
try:
instance = self.objectify(instance_type, instance)
factory(
instance_type, **{"dont_update_pools": True, **instance}
)
Session.commit()
except Exception:
info(
f"{str(instance)} could not be imported :"
f"{chr(10).join(format_exc().splitlines())}"
)
status = "Partial import (see logs)."
try:
for name, services in workflow_services.items():
workflow = fetch("workflow", name=name)
workflow.services = [
fetch("service", name=service_name) for service_name in services
]
Session.commit()
for edge in workflow_edges:
for property in ("source", "destination", "workflow"):
edge[property] = fetch("service", name=edge[property]).id
factory("workflow_edge", **edge)
Session.commit()
for service in fetch_all("service"):
service.set_name()
if not skip_update_pools_after_import:
for pool in fetch_all("pool"):
pool.compute_pool()
self.log("info", status)
except Exception:
info(chr(10).join(format_exc().splitlines()))
status = "Partial import (see logs)."
return status
def import_topology(self, **kwargs: Any) -> str:
file = kwargs["file"]
if kwargs["replace"]:
delete_all("Device")
Session.commit()
if self.allowed_file(secure_filename(file.filename), {"xls", "xlsx"}):
result = self.topology_import(file)
info("Inventory import: Done.")
return result
def database_deletion(self, **kwargs: Any) -> None:
delete_all(*kwargs["deletion_types"])
clear_logs_date = kwargs["clear_logs_date"]
if clear_logs_date:
clear_date = datetime.strptime(clear_logs_date, "%d/%m/%Y %H:%M:%S")
for job in fetch_all("Job"):
job.logs = {
date: log
for date, log in job.logs.items()
if datetime.strptime(date, "%Y-%m-%d-%H:%M:%S.%f") > clear_date
}
def test_manual_object_creation(user_client: FlaskClient) -> None:
delete_all("Device", "Link")
for icon in device_icons:
for description in ("desc1", "desc2"):
obj_dict = define_device(icon, description)
user_client.post("/update/device", data=obj_dict)
devices = fetch_all("Device")
for source in devices[:3]:
for destination in devices[:3]:
obj_dict = define_link(source.id, destination.id)
user_client.post("/update/link", data=obj_dict)
assert len(fetch_all("Device")) == 16
assert len(fetch_all("Link")) == 9
def test_manual_object_creation(user_client):
delete_all("device", "link")
for number in range(10):
for description in ("desc1", "desc2"):
obj_dict = define_device(number, description)
user_client.post("/update/device", data=obj_dict)
devices = fetch_all("device")
for source in devices[:3]:
for destination in devices[:3]:
obj_dict = define_link(source.id, destination.id)
user_client.post("/update/link", data=obj_dict)
assert len(fetch_all("device")) == 10
assert len(fetch_all("link")) == 9
def migration_import(self, **kwargs: Any) -> str:
status, types = "Import successful.", kwargs["import_export_types"]
if kwargs.get("empty_database_before_import", False):
for type in types:
delete_all(type)
Session.commit()
workflow_edges: list = []
for cls in types:
path = (self.path / "projects" / "migrations" / kwargs["name"] /
f"{cls}.yaml")
with open(path, "r") as migration_file:
objects = yaml.load(migration_file)
if cls == "Workflow":
workflow_jobs = {
workflow["name"]: workflow.pop("jobs")
for workflow in objects
}
if cls == "WorkflowEdge":
workflow_edges = deepcopy(objects)
if cls == "Service":
objects.sort(key=lambda s: s["type"] == "IterationService")
for obj in objects:
obj_cls = obj.pop("type") if cls == "Service" else cls
obj = self.objectify(obj_cls, obj)
try:
factory(obj_cls, **obj)
Session.commit()
except Exception as e:
info(f"{str(obj)} could not be imported ({str(e)})")
if cls in ("Service", "Workflow"):
Session.commit()
status = "Partial import (see logs)."
if cls not in ("Service", "Workflow"):
Session.commit()
for name, jobs in workflow_jobs.items():
fetch("Workflow",
name=name).jobs = [fetch("Job", name=name) for name in jobs]
Session.commit()
for edge in workflow_edges:
for property in ("source", "destination", "workflow"):
edge[property] = fetch("Job", name=edge[property]).id
factory("WorkflowEdge", **edge)
Session.commit()
return status
def migration_import(self, folder="migrations", **kwargs):
status, models = "Import successful.", kwargs["import_export_types"]
if kwargs.get("empty_database_before_import", False):
for model in models:
delete_all(model)
Session.commit()
workflow_edges, workflow_services = [], {}
folder_path = self.path / "projects" / folder / kwargs["name"]
for model in models:
path = folder_path / f"{model}.yaml"
if not path.exists():
continue
with open(path, "r") as migration_file:
instances = yaml.load(migration_file)
if model == "workflow_edge":
workflow_edges = deepcopy(instances)
continue
for instance in instances:
instance_type = (instance.pop("type")
if model == "service" else model)
if instance_type == "workflow":
workflow_services[instance["name"]] = instance.pop(
"services")
instance = self.objectify(instance_type, instance)
try:
factory(instance_type, **instance)
Session.commit()
except Exception as e:
info(
f"{str(instance)} could not be imported ({str(e)})"
)
status = "Partial import (see logs)."
for name, services in workflow_services.items():
fetch("workflow", name=name).services = [
fetch("service", name=service_name)
for service_name in services
]
Session.commit()
for edge in workflow_edges:
for property in ("source", "destination", "workflow"):
edge[property] = fetch("service", name=edge[property]).id
factory("workflow_edge", **edge)
Session.commit()
return status
def database_deletion(self, **kwargs):
delete_all(*kwargs["deletion_types"])
def database_deletion(self, **kwargs: Any) -> None:
delete_all(*kwargs["deletion_types"])
