async def create_job_for_agent(stack_instance,
action,
document_manager,
redis,
first_run=True,
force_delete=False):
"""creates jobs and puts them on the right agent queue"""
logger.debug(
f"For stack_instance '{stack_instance}' and action '{action}'")
success = True
success = await create_job_per_service(stack_instance.services,
document_manager, action, redis,
stack_instance)
if action == "delete" and (success or force_delete):
document_manager.delete_stack_instance(stack_instance.name)
elif not success and first_run and config.settings.rollback_enabled:
snapshot_document = get_snapshot_manager().restore_latest_snapshot(
"stack_instance", stack_instance.name)
stack_instance = StackInstance.parse_obj(snapshot_document["snapshot"])
await create_job_for_agent(stack_instance,
action,
document_manager,
redis,
first_run=False)
def _create_stack_instance(
self, item, opa_decision,
stack_infrastructure_template: StackInfrastructureTemplate,
opa_service_params):
"""
function for creating the stack instance object
"""
stack_instance_doc = StackInstance(
name=item.stack_instance_name,
stack_infrastructure_template=item.stack_infrastructure_template,
stack_application_template=item.stack_application_template)
stack_instance_doc.instance_params = item.params
stack_instance_doc.service_params = item.service_params
stack_instance_doc.instance_secrets = item.secrets
services = OrderedDict()
stack_instance_statuses = []
for svc, opa_result in opa_decision.items():
# if a svc doesnt have a result raise an error cause we cant resolve it
svc_doc = self.document_manager.get_service(opa_result['service'])
service_definitions = []
for infra_target in opa_result['targets']:
infra_target_counter = 1
service_definition = self.add_service_definition(
infra_target, infra_target_counter, item,
opa_service_params, stack_infrastructure_template,
stack_instance_statuses, svc, svc_doc)
service_definitions.append(service_definition)
infra_target_counter += 1
services[svc] = service_definitions
stack_instance_doc.services = services
stack_instance_doc.status = stack_instance_statuses
return stack_instance_doc
def get_stack_instance(self, stack_instance_name):
"""gets a StackInstance Object from the store"""
store_response = self.store.get(type="stack_instance",
name=stack_instance_name,
category="items")
if store_response.status_code == 404:
logger.debug("not found returning none")
return None
stack_instance = StackInstance.parse_obj(store_response.content)
return stack_instance
def _update_stack_instance(self, stack_instance: StackInstance, item,
opa_service_params, service_targets):
"""
This method takes a stack instance and an item
which contains the extra parameters and secrets
"""
stack_infr_template = self.document_manager.get_stack_infrastructure_template(
stack_instance.stack_infrastructure_template)
stack_infrastructure_template = self._update_infr_capabilities(
stack_infr_template, "yes")
stack_instance.instance_params = {
**stack_instance.instance_params,
**item.params
}
stack_instance.instance_secrets = {
**stack_instance.instance_secrets,
**item.secrets
}
stack_instance.service_params = {
**stack_instance.service_params,
**item.service_params
}
if "stackl_groups" in item.params:
stack_instance.groups = item.params["stackl_groups"]
stack_instance_statuses = []
for svc, service_definitions in stack_instance.services.items():
svc_doc = self.document_manager.get_service(svc)
for count, service_definition in enumerate(service_definitions):
if service_targets and not service_definition.infrastructure_target in \
service_targets["result"]["services"][svc]:
return "Update impossible. Target in service definition not in service_targets"
service_definition = self.update_service_definition(
count, item, opa_service_params, service_definition,
stack_infrastructure_template, stack_instance,
stack_instance_statuses, svc)
stack_instance.services[svc][count] = service_definition
if service_targets:
if len(service_targets["result"]["services"][svc]) > len(
service_definitions):
start_index = len(service_targets["result"]["services"][svc]) \
- len(service_definitions)
for i in range(
start_index,
len(service_targets["result"]["services"][svc])):
service_definition = self.add_service_definition(
service_targets["result"]["services"][svc][i],
i + 1, item, opa_service_params,
stack_infrastructure_template,
stack_instance_statuses, svc, svc_doc)
service_definitions.append(service_definition)
stack_instance.status = stack_instance_statuses
return stack_instance
def restore_snapshot(
name: str,
background_tasks: BackgroundTasks,
document_manager: DocumentManager = Depends(get_document_manager),
snapshot_manager: SnapshotManager = Depends(get_snapshot_manager),
redis=Depends(get_redis)):
"""
Restore the latest or optionally the given number most recent snapshot
of the doc with the given type_name and name
"""
logger.info(
f"[RestoreSnapshot POST] API POST request for doc with '{name}'")
snapshot_document = snapshot_manager.restore_snapshot(name)
if snapshot_document['snapshot']["type"] == "stack_instance":
stack_instance = StackInstance.parse_obj(snapshot_document["snapshot"])
background_tasks.add_task(create_job_for_agent, stack_instance,
"update", document_manager, redis)
return {"result": "stack instance restored, restoring in progress"}
return {"result": f"snapshot {name} restored"}
def _update_stack_instance(self, stack_instance: StackInstance,
item: StackInstanceUpdate, opa_service_params,
service_targets):
"""
This method takes a stack instance and an item
which contains the extra parameters and secrets
"""
if item.stages:
stack_instance.stages = item.stages
stack_infr_template = self.document_manager.get_stack_infrastructure_template(
stack_instance.stack_infrastructure_template)
stack_infrastructure_template = self._update_infr_capabilities(
stack_infr_template, "yes")
stack_instance.instance_params = {
**stack_instance.instance_params,
**item.params
}
stack_instance.instance_secrets = {
**stack_instance.instance_secrets,
**item.secrets
}
stack_instance.service_params = {
**stack_instance.service_params,
**item.service_params
}
stack_instance.service_secrets = {
**stack_instance.service_secrets,
**item.service_secrets
}
if "stackl_groups" in item.params:
stack_instance.groups = item.params["stackl_groups"]
stack_instance_statuses = []
new_service_definitions = {}
# Create copy because we are looping over it and changing things
stack_instances_services_copy = stack_instance.services.copy()
for svc, service_definitions in stack_instances_services_copy.items():
for count, service_definition in enumerate(service_definitions):
svc_doc = self.document_manager.get_service(
service_definition.service)
if svc in service_targets and not service_definition.infrastructure_target in \
service_targets[svc]['targets']:
return "Update impossible. Target in service definition not in service_targets"
service_definition = self.update_service_definition(
count, item, opa_service_params, service_definition,
stack_infrastructure_template, stack_instance,
stack_instance_statuses, svc, svc_doc)
stack_instance.services[svc][count] = service_definition
# Check if replica count increased
if svc in item.replicas and item.replicas[svc] > len(
service_definitions):
start_index = len(service_targets[svc]["targets"]) \
- len(service_definitions)
if start_index < 1:
return f"Can't add more replicas cause there are not enough extra targets for {svc}"
# Get the service doc, but I really dont like this way:
svc_doc = self.document_manager.get_service(
service_definitions[0].service)
for i in range(start_index,
len(service_targets[svc]["targets"])):
service_definition = self.add_service_definition(
service_targets[svc]["targets"][i], i + 1, item,
opa_service_params, stack_infrastructure_template,
stack_instance_statuses, svc, svc_doc)
if not svc in new_service_definitions:
new_service_definitions[svc] = []
new_service_definitions[svc].append(service_definition)
for service in item.services:
if service.name not in stack_instance.services:
svc_doc = self.document_manager.get_service(service.service)
service_definition = self.add_service_definition(
service_targets[service.name]["targets"][0], 0, item,
opa_service_params, stack_infrastructure_template,
stack_instance_statuses, service.name, svc_doc)
new_service_definitions[service.name] = [service_definition]
stack_instance.services = {
**stack_instance.services,
**new_service_definitions
}
stack_instance.status = stack_instance_statuses
return stack_instance
async def create_job_for_agent(stack_instance,
action,
document_manager,
redis,
first_run=True,
force_delete=False):
"""creates jobs and puts them on the right agent queue"""
logger.debug(
f"For stack_instance '{stack_instance}' and action '{action}'")
success = True
sat = document_manager.get_stack_application_template(
stack_instance.stack_application_template)
for service_name in sat.services:
service_doc = document_manager.get_document(type="service",
name=service_name)
functional_requirements = service_doc["functional_requirements"]
if action == "delete":
functional_requirements = reversed(functional_requirements)
for fr in functional_requirements:
fr_doc = document_manager.get_functional_requirement(fr)
fr_jobs = []
for service_definition in stack_instance.services[service_name]:
infrastructure_target = service_definition.infrastructure_target
cloud_provider = service_definition.cloud_provider
logger.debug(
f"Retrieved fr '{fr_doc}' from service_doc '{service_doc}'"
)
invoc = {}
invoc['action'] = action
invoc['functional_requirement'] = fr
invoc['image'] = fr_doc.invocation[cloud_provider].image
invoc['before_command'] = fr_doc.invocation[
cloud_provider].before_command
invoc['infrastructure_target'] = infrastructure_target
invoc['stack_instance'] = stack_instance.name
invoc['tool'] = fr_doc.invocation[cloud_provider].tool
invoc['service'] = service_name
invoc["hosts"] = service_definition.hosts
logger.debug("Appending job")
job = await redis.enqueue_job(
"invoke_automation",
invoc,
_queue_name=service_definition.agent)
fr_jobs.append(asyncio.create_task(job.result(timeout=7200)))
if fr_doc.as_group:
logger.debug("running as group")
break
for fr_job in asyncio.as_completed(fr_jobs):
automation_result = await fr_job
await update_status(automation_result, document_manager,
stack_instance)
if automation_result["status"] == "FAILED":
success = False
if not success:
logger.debug("Not all fr's succeeded, stopping execution")
break
logger.debug("tasks executed")
logger.debug(f"rollback_enabled: {config.settings.rollback_enabled}")
if action == "delete" and (success or force_delete):
document_manager.delete_stack_instance(stack_instance.name)
elif not success and first_run and config.settings.rollback_enabled:
snapshot_document = get_snapshot_manager().restore_latest_snapshot(
"stack_instance", stack_instance.name)
stack_instance = StackInstance.parse_obj(snapshot_document["snapshot"])
await create_job_for_agent(stack_instance,
action,
document_manager,
redis,
first_run=False)