async def create_dryrun(self, env: data.Environment, version_id: int, model: data.ConfigurationModel) -> data.DryRun:
# fetch all resource in this cm and create a list of distinct agents
rvs = await data.Resource.get_list(model=version_id, environment=env.id)
# Create a dryrun document
dryrun = await data.DryRun.create(environment=env.id, model=version_id, todo=len(rvs), total=len(rvs))
agents = await data.ConfigurationModel.get_agents(env.id, version_id)
await self.autostarted_agent_manager._ensure_agents(env, agents)
agents_down = []
for agent in agents:
client = self.agent_manager.get_agent_client(env.id, agent)
if client is not None:
self.add_background_task(client.do_dryrun(env.id, dryrun.id, agent, version_id))
else:
agents_down.append(agent)
LOGGER.warning("Agent %s from model %s in env %s is not available for a dryrun", agent, version_id, env.id)
# Mark the resources in an undeployable state as done
async with self.dryrun_lock:
undeployable_ids = await model.get_undeployable()
undeployable_version_ids = [ResourceVersionIdStr(rid + ",v=%s" % version_id) for rid in undeployable_ids]
undeployable = await data.Resource.get_resources(environment=env.id, resource_version_ids=undeployable_version_ids)
await self._save_resources_without_changes_to_dryrun(
dryrun_id=dryrun.id, resources=undeployable, diff_status=ResourceDiffStatus.undefined
)
skip_undeployable_ids = await model.get_skipped_for_undeployable()
skip_undeployable_version_ids = [ResourceVersionIdStr(rid + ",v=%s" % version_id) for rid in skip_undeployable_ids]
skipundeployable = await data.Resource.get_resources(
environment=env.id, resource_version_ids=skip_undeployable_version_ids
)
await self._save_resources_without_changes_to_dryrun(
dryrun_id=dryrun.id, resources=skipundeployable, diff_status=ResourceDiffStatus.skipped_for_undefined
)
resources_with_agents_down = [
res
for res in rvs
if res.resource_version_id not in undeployable_version_ids
and res.resource_version_id not in skip_undeployable_version_ids
and res.agent in agents_down
]
await self._save_resources_without_changes_to_dryrun(
dryrun_id=dryrun.id, resources=resources_with_agents_down, diff_status=ResourceDiffStatus.agent_down
)
return dryrun
async def test_log_deploy_start(server, client, environment, clienthelper,
agent, resource_deployer):
"""
Ensure that a message is logged when starting a deploy.
"""
# Version 1
version = await clienthelper.get_version()
rid_r1 = ResourceIdStr("std::File[agent1,path=/etc/file1]")
rvid_r1_v1 = ResourceVersionIdStr(f"{rid_r1},v={version}")
resources = [
{
"path": "/etc/file1",
"id": rvid_r1_v1,
"requires": [],
"purged": False,
"send_event": False
},
]
await clienthelper.put_version_simple(resources, version)
# Start new deployment for r1
await resource_deployer.start_deployment(rvid=rvid_r1_v1)
result = await client.resource_logs(environment, rid_r1)
assert result.code == 200
deploy_started_message = next(
(log_message for log_message in result.result["data"]
if "Resource deploy started on agent" in log_message["msg"]), None)
assert deploy_started_message
async def create_resource_in_multiple_versions(
environment: uuid.UUID,
path: str,
version_attributes_map: Dict[int, Dict[str, object]],
agent: str = "internal",
resource_type: str = "std::File",
status: ResourceState = ResourceState.deployed,
):
key = f"{resource_type}[{agent},path={path}]"
for version, attributes in version_attributes_map.items():
attributes["requires"] = [
f"{req},v={version}" for req in attributes.get("requires", [])
]
res = data.Resource.new(
environment=environment,
resource_version_id=ResourceVersionIdStr(f"{key},v={version}"),
attributes={
**attributes,
**{
"path": path
}
},
status=status,
last_deploy=datetime.datetime.now(),
)
await res.insert()
async def test_resource_list_no_released_version(server, client):
"""Test that if there are no released versions of a resource, the result set is empty"""
project = data.Project(name="test")
await project.insert()
env = data.Environment(name="dev", project=project.id, repo_url="", repo_branch="")
await env.insert()
version = 1
cm = data.ConfigurationModel(
environment=env.id,
version=version,
date=datetime.now(),
total=1,
released=False,
version_info={},
)
await cm.insert()
path = f"/etc/file{1}"
key = f"std::File[agent1,path={path}]"
res1_v1 = data.Resource.new(
environment=env.id, resource_version_id=ResourceVersionIdStr(f"{key},v={version}"), attributes={"path": path}
)
await res1_v1.insert()
result = await client.resource_list(env.id)
assert result.code == 200
assert len(result.result["data"]) == 0
async def test_events_resource_without_dependencies(server, client,
environment, clienthelper,
agent, resource_deployer):
"""
Ensure that events are captured across versions.
"""
# Version 1
version = await clienthelper.get_version()
rvid_r1_v1 = ResourceVersionIdStr(
f"std::File[agent1,path=/etc/file1],v={version}")
resources = [
{
"path": "/etc/file1",
"id": rvid_r1_v1,
"requires": [],
"purged": False,
"send_event": False
},
]
await clienthelper.put_version_simple(resources, version)
# Start new deployment for r1
await resource_deployer.start_deployment(rvid=rvid_r1_v1)
result = await agent._client.get_resource_events(tid=environment,
rvid=rvid_r1_v1)
assert result.code == 200
assert len(result.result["data"]) == 0
result = await agent._client.resource_did_dependency_change(
tid=environment, rvid=rvid_r1_v1)
assert result.code == 200
assert not result.result["data"]
async def create_resource(
agent: str, path: str, resource_type: str, status: ResourceState, versions: List[int], environment: UUID = env.id
):
for version in versions:
key = f"{resource_type}[{agent},path={path}]"
res = data.Resource.new(
environment=environment,
resource_version_id=ResourceVersionIdStr(f"{key},v={version}"),
attributes={"path": path},
status=status,
)
await res.insert()
async def create_resource(
path: str,
status: ResourceState,
version: int,
attributes: Dict[str, object],
agent: str = "internal",
resource_type: str = "std::File",
):
key = f"{resource_type}[{agent},path={path}]"
res = data.Resource.new(
environment=environment,
resource_version_id=ResourceVersionIdStr(f"{key},v={version}"),
attributes={
**attributes,
**{
"path": path
}
},
status=status,
last_deploy=datetime.datetime.now(),
)
await res.insert()
return res
async def dryrun_request(self, env: data.Environment,
version_id: int) -> Apireturn:
model = await data.ConfigurationModel.get_version(environment=env.id,
version=version_id)
if model is None:
return 404, {"message": "The request version does not exist."}
# fetch all resource in this cm and create a list of distinct agents
rvs = await data.Resource.get_list(model=version_id,
environment=env.id)
# Create a dryrun document
dryrun = await data.DryRun.create(environment=env.id,
model=version_id,
todo=len(rvs),
total=len(rvs))
agents = await data.ConfigurationModel.get_agents(env.id, version_id)
await self.autostarted_agent_manager._ensure_agents(env, agents)
for agent in agents:
client = self.agent_manager.get_agent_client(env.id, agent)
if client is not None:
self.add_background_task(
client.do_dryrun(env.id, dryrun.id, agent, version_id))
else:
LOGGER.warning(
"Agent %s from model %s in env %s is not available for a dryrun",
agent, version_id, env.id)
# Mark the resources in an undeployable state as done
async with self.dryrun_lock:
undeployable_ids = await model.get_undeployable()
undeployable_version_ids = [
ResourceVersionIdStr(rid + ",v=%s" % version_id)
for rid in undeployable_ids
]
undeployable = await data.Resource.get_resources(
environment=env.id,
resource_version_ids=undeployable_version_ids)
for res in undeployable:
parsed_id = Id.parse_id(res.resource_version_id)
payload = {
"changes": {},
"id_fields": {
"entity_type": res.resource_type,
"agent_name": res.agent,
"attribute": parsed_id.attribute,
"attribute_value": parsed_id.attribute_value,
"version": res.model,
},
"id": res.resource_version_id,
}
await data.DryRun.update_resource(dryrun.id,
res.resource_version_id,
payload)
skip_undeployable_ids = await model.get_skipped_for_undeployable()
skip_undeployable_version_ids = [
ResourceVersionIdStr(rid + ",v=%s" % version_id)
for rid in skip_undeployable_ids
]
skipundeployable = await data.Resource.get_resources(
environment=env.id,
resource_version_ids=skip_undeployable_version_ids)
for res in skipundeployable:
parsed_id = Id.parse_id(res.resource_version_id)
payload = {
"changes": {},
"id_fields": {
"entity_type": res.resource_type,
"agent_name": res.agent,
"attribute": parsed_id.attribute,
"attribute_value": parsed_id.attribute_value,
"version": res.model,
},
"id": res.resource_version_id,
}
await data.DryRun.update_resource(dryrun.id,
res.resource_version_id,
payload)
return 200, {"dryrun": dryrun}
async def release_version(
self,
env: data.Environment,
version_id: int,
push: bool,
agent_trigger_method: Optional[const.AgentTriggerMethod] = None,
) -> Apireturn:
model = await data.ConfigurationModel.get_version(env.id, version_id)
if model is None:
return 404, {"message": "The request version does not exist."}
await model.update_fields(released=True,
result=const.VersionState.deploying)
if model.total == 0:
await model.mark_done()
return 200, {"model": model}
# Already mark undeployable resources as deployed to create a better UX (change the version counters)
undep = await model.get_undeployable()
undep_ids = [
ResourceVersionIdStr(rid + ",v=%s" % version_id) for rid in undep
]
now = datetime.datetime.now().astimezone()
# not checking error conditions
await self.resource_service.resource_action_update(
env,
undep_ids,
action_id=uuid.uuid4(),
started=now,
finished=now,
status=const.ResourceState.undefined,
action=const.ResourceAction.deploy,
changes={},
messages=[],
change=const.Change.nochange,
send_events=False,
)
skippable = await model.get_skipped_for_undeployable()
skippable_ids = [
ResourceVersionIdStr(rid + ",v=%s" % version_id)
for rid in skippable
]
# not checking error conditions
await self.resource_service.resource_action_update(
env,
skippable_ids,
action_id=uuid.uuid4(),
started=now,
finished=now,
status=const.ResourceState.skipped_for_undefined,
action=const.ResourceAction.deploy,
changes={},
messages=[],
change=const.Change.nochange,
send_events=False,
)
if push:
# fetch all resource in this cm and create a list of distinct agents
agents = await data.ConfigurationModel.get_agents(
env.id, version_id)
await self.autostarted_agent_manager._ensure_agents(env, agents)
for agent in agents:
client = self.agentmanager_service.get_agent_client(
env.id, agent)
if client is not None:
if not agent_trigger_method:
env_agent_trigger_method = await env.get(
ENVIRONMENT_AGENT_TRIGGER_METHOD)
incremental_deploy = env_agent_trigger_method == const.AgentTriggerMethod.push_incremental_deploy
else:
incremental_deploy = agent_trigger_method is const.AgentTriggerMethod.push_incremental_deploy
self.add_background_task(
client.trigger(env.id, agent, incremental_deploy))
else:
LOGGER.warning(
"Agent %s from model %s in env %s is not available for a deploy",
agent, version_id, env.id)
return 200, {"model": model}
async def put_version(
self,
env: data.Environment,
version: int,
resources: List[JsonType],
resource_state: Dict[ResourceIdStr, const.ResourceState],
unknowns: List[Dict[str, PrimitiveTypes]],
version_info: JsonType,
compiler_version: Optional[str] = None,
) -> Apireturn:
"""
:param resources: a list of serialized resources
:param unknowns: dict with the following structure
{
"resource": ResourceIdStr,
"parameter": str,
"source": str
}
:param version_info:
:param compiler_version:
:return:
"""
if not compiler_version:
raise BadRequest(
"Older compiler versions are no longer supported, please update your compiler"
)
if version > env.last_version:
raise BadRequest(
f"The version number used is {version} "
f"which is higher than the last outstanding reservation {env.last_version}"
)
if version <= 0:
raise BadRequest(
f"The version number used ({version}) is not positive")
started = datetime.datetime.now().astimezone()
agents = set()
# lookup for all RV's, lookup by resource id
rv_dict: Dict[ResourceVersionIdStr, data.Resource] = {}
# reverse dependency tree, Resource.provides [:] -- Resource.requires as resource_id
provides_tree: Dict[str, List[str]] = defaultdict(lambda: [])
# list of all resources which have a cross agent dependency, as a tuple, (dependant,requires)
cross_agent_dep = []
# list of all resources which are undeployable
undeployable: List[data.Resource] = []
resource_objects = []
resource_version_ids = []
for res_dict in resources:
res_obj = data.Resource.new(env.id, res_dict["id"])
if res_obj.resource_id in resource_state:
res_obj.status = const.ResourceState[resource_state[
res_obj.resource_id]]
if res_obj.status in const.UNDEPLOYABLE_STATES:
undeployable.append(res_obj)
# collect all agents
agents.add(res_obj.agent)
attributes = {}
for field, value in res_dict.items():
if field != "id":
attributes[field] = value
res_obj.attributes = attributes
resource_objects.append(res_obj)
resource_version_ids.append(res_obj.resource_version_id)
rv_dict[res_obj.resource_id] = res_obj
# find cross agent dependencies
agent = res_obj.agent
resc_id = res_obj.resource_id
if "requires" not in attributes:
LOGGER.warning(
"Received resource without requires attribute (%s)" %
res_obj.resource_id)
else:
for req in attributes["requires"]:
rid = Id.parse_id(req)
provides_tree[rid.resource_str()].append(resc_id)
if rid.get_agent_name() != agent:
# it is a CAD
cross_agent_dep.append((res_obj, rid))
# hook up all CADs
for f, t in cross_agent_dep:
res_obj = rv_dict[t.resource_str()]
res_obj.provides.append(f.resource_version_id)
# detect failed compiles
def safe_get(input: JsonType, key: str, default: object) -> object:
if not isinstance(input, dict):
return default
if key not in input:
return default
return input[key]
metadata: JsonType = safe_get(version_info, const.EXPORT_META_DATA, {})
compile_state = safe_get(metadata, const.META_DATA_COMPILE_STATE, "")
failed = compile_state == const.Compilestate.failed
resources_to_purge: List[data.Resource] = []
if not failed and (await env.get(PURGE_ON_DELETE)):
# search for deleted resources (purge_on_delete)
resources_to_purge = await data.Resource.get_deleted_resources(
env.id, version, set(rv_dict.keys()))
previous_requires = {}
for res in resources_to_purge:
LOGGER.warning("Purging %s, purged resource based on %s" %
(res.resource_id, res.resource_version_id))
attributes = res.attributes.copy()
attributes["purged"] = True
attributes["requires"] = []
res_obj = data.Resource.new(
env.id,
resource_version_id=ResourceVersionIdStr(
"%s,v=%s" % (res.resource_id, version)),
attributes=attributes,
)
resource_objects.append(res_obj)
previous_requires[
res_obj.resource_id] = res.attributes["requires"]
resource_version_ids.append(res_obj.resource_version_id)
agents.add(res_obj.agent)
rv_dict[res_obj.resource_id] = res_obj
# invert dependencies on purges
for res_id, requires in previous_requires.items():
res_obj = rv_dict[res_id]
for require in requires:
req_id = Id.parse_id(require)
if req_id.resource_str() in rv_dict:
req_res = rv_dict[req_id.resource_str()]
req_res.attributes["requires"].append(
res_obj.resource_version_id)
res_obj.provides.append(req_res.resource_version_id)
undeployable_ids: List[str] = [res.resource_id for res in undeployable]
# get skipped for undeployable
work = list(undeployable_ids)
skippeable: Set[str] = set()
while len(work) > 0:
current = work.pop()
if current in skippeable:
continue
skippeable.add(current)
work.extend(provides_tree[current])
skip_list = sorted(list(skippeable - set(undeployable_ids)))
try:
cm = data.ConfigurationModel(
environment=env.id,
version=version,
date=datetime.datetime.now().astimezone(),
total=len(resources),
version_info=version_info,
undeployable=undeployable_ids,
skipped_for_undeployable=skip_list,
)
await cm.insert()
except asyncpg.exceptions.UniqueViolationError:
raise ServerError(
"The given version is already defined. Versions should be unique."
)
await data.Resource.insert_many(resource_objects)
await cm.update_fields(total=cm.total + len(resources_to_purge))
for uk in unknowns:
if "resource" not in uk:
uk["resource"] = ""
if "metadata" not in uk:
uk["metadata"] = {}
up = data.UnknownParameter(
resource_id=uk["resource"],
name=uk["parameter"],
source=uk["source"],
environment=env.id,
version=version,
metadata=uk["metadata"],
)
await up.insert()
for agent in agents:
await self.agentmanager_service.ensure_agent_registered(env, agent)
# Don't log ResourceActions without resource_version_ids, because
# no API call exists to retrieve them.
if resource_version_ids:
now = datetime.datetime.now().astimezone()
log_line = data.LogLine.log(
logging.INFO,
"Successfully stored version %(version)d",
version=version)
self.resource_service.log_resource_action(env.id,
resource_version_ids,
logging.INFO, now,
log_line.msg)
ra = data.ResourceAction(
environment=env.id,
version=version,
resource_version_ids=resource_version_ids,
action_id=uuid.uuid4(),
action=const.ResourceAction.store,
started=started,
finished=now,
messages=[log_line],
)
await ra.insert()
LOGGER.debug("Successfully stored version %d", version)
self.resource_service.clear_env_cache(env)
auto_deploy = await env.get(data.AUTO_DEPLOY)
if auto_deploy:
LOGGER.debug("Auto deploying version %d", version)
push_on_auto_deploy = cast(bool, await
env.get(data.PUSH_ON_AUTO_DEPLOY))
agent_trigger_method_on_autodeploy = cast(
str, await env.get(data.AGENT_TRIGGER_METHOD_ON_AUTO_DEPLOY))
agent_trigger_method_on_autodeploy = const.AgentTriggerMethod[
agent_trigger_method_on_autodeploy]
await self.release_version(env, version, push_on_auto_deploy,
agent_trigger_method_on_autodeploy)
return 200
def version_report(client: Client, environment: str, version: str,
show_detailed_report: bool) -> None:
tid = client.to_environment_id(environment)
result = client.do_request("get_version",
arguments=dict(tid=tid,
id=version,
include_logs=True))
if not result:
return
agents: Dict[str, Dict[str, List[str]]] = defaultdict(
lambda: defaultdict(lambda: []))
for res in result["resources"]:
if len(res["actions"]) > 0 or show_detailed_report:
agents[res["agent"]][res["resource_type"]].append(res)
for agent in sorted(agents.keys()):
click.echo(click.style("Agent: %s" % agent, bold=True))
click.echo("=" * 72)
for t in sorted(agents[agent].keys()):
parsed_resource_version_id = Id.parse_id(
ResourceVersionIdStr(
agents[agent][t][0]["resource_version_id"]))
click.echo(
click.style("Resource type:", bold=True) +
"{type} ({attr})".format(
type=t, attr=parsed_resource_version_id.attribute))
click.echo("-" * 72)
for res in agents[agent][t]:
parsed_id = Id.parse_id(res["resource_version_id"])
click.echo((click.style(parsed_id.attribute_value, bold=True) +
" (#actions=%d)") % len(res["actions"]))
# for dryrun show only the latest, for deploy all
if not result["model"]["released"]:
if len(res["actions"]) > 0:
action = res["actions"][0]
click.echo("* last check: %s" % action["timestamp"])
click.echo("* result: %s" %
("error" if action["level"] != "INFO" else
"success"))
if len(action["data"]) == 0:
click.echo("* no changes")
else:
click.echo("* changes:")
for field in sorted(action["data"].keys()):
values = action["data"][field]
if field == "hash":
click.echo(" - content:")
diff_value = client.do_request(
"diff",
arguments=dict(a=values[0],
b=values[1]))
click.echo(" " +
" ".join(diff_value["diff"]))
else:
click.echo(" - %s:" % field)
click.echo(
" " +
click.style("from:", bold=True) +
" %s" % values[0])
click.echo(" " +
click.style("to:", bold=True) +
" %s" % values[1])
click.echo("")
click.echo("")
else:
pass
click.echo("")
def validate_resource_version_id(ctx: click.Context,
option: Union[click.Option, click.Parameter],
value: str) -> ResourceVersionIdStr:
if not Id.is_resource_version_id(value):
raise click.BadParameter(value)
return ResourceVersionIdStr(value)
async def env_with_facts(environment,
client) -> Tuple[str, List[str], List[str]]:
env_id = uuid.UUID(environment)
version = 1
await data.ConfigurationModel(
environment=env_id,
version=version,
date=datetime.now(),
total=1,
released=True,
version_info={},
).insert()
path = "/etc/file1"
resource_id = f"std::File[agent1,path={path}]"
res1_v1 = data.Resource.new(
environment=env_id,
resource_version_id=ResourceVersionIdStr(f"{resource_id},v={version}"),
attributes={"path": path})
await res1_v1.insert()
path = "/etc/file2"
resource_id_2 = f"std::File[agent1,path={path}]"
await data.Resource.new(
environment=env_id,
resource_version_id=ResourceVersionIdStr(
f"{resource_id_2},v={version}"),
attributes={
"path": path
},
).insert()
resource_id_3 = "std::File[agent1,path=/etc/file3]"
path = "/tmp/filex"
resource_id_4 = f"std::File[agent1,path={path}]"
await data.Resource.new(
environment=env_id,
resource_version_id=ResourceVersionIdStr(
f"{resource_id_4},v={version}"),
attributes={
"path": path
},
).insert()
async def insert_param(
name: str,
resource_id: Optional[str] = None,
source: str = "fact",
updated: Optional[datetime] = None,
metadata: Optional[Dict[str, str]] = None,
) -> uuid.UUID:
param_id = uuid.uuid4()
await data.Parameter(
id=param_id,
name=name,
value="42",
environment=env_id,
source=source,
resource_id=resource_id,
updated=updated,
metadata=metadata,
).insert()
return param_id
param_id_1 = await insert_param("param",
resource_id=resource_id,
updated=datetime.now())
await insert_param("param2", resource_id=resource_id)
param_id_3 = await insert_param(
"param_for_other_resource",
resource_id_2,
updated=datetime.now(),
metadata={"very_important_metadata": "123"})
for i in range(5):
await insert_param(
f"param_for_new_resource{i}",
resource_id_4,
updated=datetime.now() if i % 2 else None,
metadata={"new_metadata": "42"} if i % 3 else None,
)
await insert_param("param_not_related_to_resource", source="plugin")
yield environment, [param_id_1, param_id_3], [
resource_id, resource_id_2, resource_id_3, resource_id_4
]
async def test_events_api_endpoints_basic_case(server, client, environment,
clienthelper, agent,
resource_deployer):
"""
Test whether the `get_resource_events` and the `resource_did_dependency_change`
endpoints behave as expected
"""
version = await clienthelper.get_version()
# a name that is hard to parse
rid = r"""exec::Run[agent1,command=sh -c "git _%\/ clone \"https://codis.git\" && chown -R centos:centos "]"""
rid_r1_v1 = ResourceIdStr(rid)
rvid_r1_v1 = ResourceVersionIdStr(f"{rid_r1_v1},v={version}")
rid_r2_v1 = ResourceIdStr("std::File[agent1,path=/etc/file2]")
rvid_r2_v1 = ResourceVersionIdStr(f"{rid_r2_v1},v={version}")
rid_r3_v1 = ResourceIdStr("std::File[agent1,path=/etc/file3]")
rvid_r3_v1 = ResourceVersionIdStr(f"{rid_r3_v1},v={version}")
resources = [
{
"path": "/etc/file1",
"id": rvid_r1_v1,
"requires": [rvid_r2_v1, rvid_r3_v1],
"purged": False,
"send_event": False
},
{
"path": "/etc/file2",
"id": rvid_r2_v1,
"requires": [],
"purged": False,
"send_event": False
},
{
"path": "/etc/file3",
"id": rvid_r3_v1,
"requires": [],
"purged": False,
"send_event": False
},
]
await clienthelper.put_version_simple(resources, version)
result = await agent._client.get_resource_events(tid=environment,
rvid=rvid_r1_v1)
assert result.code == 400, result.result
assert "Fetching resource events only makes sense when the resource is currently deploying" in result.result[
"message"]
result = await agent._client.resource_did_dependency_change(
tid=environment, rvid=rvid_r1_v1)
assert result.code == 400
assert "Fetching resource events only makes sense when the resource is currently deploying" in result.result[
"message"]
# Perform deployment
await resource_deployer.deploy_resource(rvid=rvid_r2_v1)
await resource_deployer.deploy_resource(rvid=rvid_r3_v1,
status=const.ResourceState.failed)
action_id = await resource_deployer.start_deployment(rvid=rvid_r1_v1)
# Verify that events exist
result = await agent._client.get_resource_events(tid=environment,
rvid=rvid_r1_v1)
assert result.code == 200
assert len(result.result["data"]) == 2
assert len(result.result["data"][rid_r2_v1]) == 1
assert result.result["data"][rid_r2_v1][0][
"action"] == const.ResourceAction.deploy
assert result.result["data"][rid_r2_v1][0][
"status"] == const.ResourceState.deployed
assert len(result.result["data"][rid_r3_v1]) == 1
assert result.result["data"][rid_r3_v1][0][
"action"] == const.ResourceAction.deploy
assert result.result["data"][rid_r3_v1][0][
"status"] == const.ResourceState.failed
result = await agent._client.resource_did_dependency_change(
tid=environment, rvid=rvid_r1_v1)
assert result.code == 200
assert result.result["data"]
# Finish first deployment
await resource_deployer.deployment_finished(rvid=rvid_r1_v1,
action_id=action_id)
# Start new deployment r1
action_id = await resource_deployer.start_deployment(rvid=rvid_r1_v1)
# Assert no events anymore
result = await agent._client.get_resource_events(tid=environment,
rvid=rvid_r1_v1)
assert result.code == 200
assert len(result.result["data"]) == 2
assert len(result.result["data"][rid_r2_v1]) == 0
assert len(result.result["data"][rid_r3_v1]) == 0
result = await agent._client.resource_did_dependency_change(
tid=environment, rvid=rvid_r1_v1)
assert result.code == 200
assert not result.result["data"]
# Finish deployment r1
await resource_deployer.deployment_finished(rvid=rvid_r1_v1,
action_id=action_id)
# Deploy r2 and r3, but no changes occurred.
await resource_deployer.deploy_resource(rvid=rvid_r2_v1,
change=const.Change.nochange)
await resource_deployer.deploy_resource(rvid=rvid_r3_v1,
change=const.Change.nochange)
# Start new deployment r1
action_id = await resource_deployer.start_deployment(rvid=rvid_r1_v1)
# Ensure events, but no reload deployment required
result = await agent._client.get_resource_events(tid=environment,
rvid=rvid_r1_v1)
assert result.code == 200
assert len(result.result["data"]) == 2
assert len(result.result["data"][rid_r2_v1]) == 1
assert result.result["data"][rid_r2_v1][0][
"action"] == const.ResourceAction.deploy
assert result.result["data"][rid_r2_v1][0][
"status"] == const.ResourceState.deployed
assert len(result.result["data"][rid_r3_v1]) == 1
assert result.result["data"][rid_r3_v1][0][
"action"] == const.ResourceAction.deploy
assert result.result["data"][rid_r3_v1][0][
"status"] == const.ResourceState.deployed
result = await agent._client.resource_did_dependency_change(
tid=environment, rvid=rvid_r1_v1)
assert result.code == 200
assert not result.result["data"]
# Finish deployment r1
await resource_deployer.deployment_finished(rvid=rvid_r1_v1,
action_id=action_id)
async def test_last_non_deploying_status_field_on_resource(
client, environment, clienthelper, resource_deployer, agent,
endpoint_to_use: str) -> None:
"""
Test whether the `last_non_deploying_status` field is updated correctly when a deployment of a resource is done.
:param endpoint_to_use: Indicates which code path should be used to report a resource action updates.
The old one (resource_action_update) or the new one (deployment_endpoint).
"""
version = await clienthelper.get_version()
rvid_r1_v1 = ResourceVersionIdStr(
f"std::File[agent1,path=/etc/file1],v={version}")
rvid_r2_v1 = ResourceVersionIdStr(
f"std::File[agent1,path=/etc/file2],v={version}")
resources = [
{
"path": "/etc/file1",
"id": rvid_r1_v1,
"requires": [],
"purged": False,
"send_event": False
},
{
"path": "/etc/file2",
"id": rvid_r2_v1,
"requires": [],
"purged": False,
"send_event": False
},
]
await clienthelper.put_version_simple(resources, version)
async def assert_status_fields(
r1_status: const.ResourceState,
r1_last_non_deploying_status: const.ResourceState,
r2_status: const.ResourceState,
r2_last_non_deploying_status: const.ResourceState,
) -> None:
db_resources = await data.Resource.get_list(environment=environment)
rvid_to_resources = {
res.resource_version_id: res
for res in db_resources
}
assert rvid_to_resources[rvid_r1_v1].status is r1_status
assert rvid_to_resources[
rvid_r1_v1].last_non_deploying_status is r1_last_non_deploying_status
assert rvid_to_resources[rvid_r2_v1].status is r2_status
assert rvid_to_resources[
rvid_r2_v1].last_non_deploying_status is r2_last_non_deploying_status
async def start_deployment(rvid: ResourceVersionIdStr) -> uuid.UUID:
if endpoint_to_use == "deployment_endpoint":
return await resource_deployer.start_deployment(rvid=rvid)
else:
action_id = uuid.uuid4()
result = await agent._client.resource_action_update(
tid=environment,
resource_ids=[rvid],
action_id=action_id,
action=const.ResourceAction.deploy,
started=datetime.datetime.now().astimezone(),
status=const.ResourceState.deploying,
)
assert result.code == 200
return action_id
async def deployment_finished(rvid: ResourceVersionIdStr,
action_id: uuid.UUID,
status: const.ResourceState) -> None:
if endpoint_to_use == "deployment_endpoint":
await resource_deployer.deployment_finished(rvid=rvid,
action_id=action_id,
status=status)
else:
now = datetime.datetime.now().astimezone()
result = await agent._client.resource_action_update(
tid=environment,
resource_ids=[rvid],
action_id=action_id,
action=const.ResourceAction.deploy,
started=now,
finished=now,
status=status,
)
assert result.code == 200
# All resources in available state
await assert_status_fields(
r1_status=const.ResourceState.available,
r1_last_non_deploying_status=const.ResourceState.available,
r2_status=const.ResourceState.available,
r2_last_non_deploying_status=const.ResourceState.available,
)
# Put R1 in deploying state
action_id_r1 = await start_deployment(rvid=rvid_r1_v1)
await assert_status_fields(
r1_status=const.ResourceState.deploying,
r1_last_non_deploying_status=const.ResourceState.available,
r2_status=const.ResourceState.available,
r2_last_non_deploying_status=const.ResourceState.available,
)
# R1 finished deployment + R2 start deployment
await deployment_finished(rvid=rvid_r1_v1,
action_id=action_id_r1,
status=const.ResourceState.deployed)
action_id_r2 = await start_deployment(rvid=rvid_r2_v1)
await assert_status_fields(
r1_status=const.ResourceState.deployed,
r1_last_non_deploying_status=const.ResourceState.deployed,
r2_status=const.ResourceState.deploying,
r2_last_non_deploying_status=const.ResourceState.available,
)
# R1 start deployment + R2 skipped
action_id_r1 = await start_deployment(rvid=rvid_r1_v1)
await deployment_finished(rvid=rvid_r2_v1,
action_id=action_id_r2,
status=const.ResourceState.skipped)
await assert_status_fields(
r1_status=const.ResourceState.deploying,
r1_last_non_deploying_status=const.ResourceState.deployed,
r2_status=const.ResourceState.skipped,
r2_last_non_deploying_status=const.ResourceState.skipped,
)
# R1 failed + R2 start deployment
await deployment_finished(rvid=rvid_r1_v1,
action_id=action_id_r1,
status=const.ResourceState.failed)
await start_deployment(rvid=rvid_r2_v1)
await assert_status_fields(
r1_status=const.ResourceState.failed,
r1_last_non_deploying_status=const.ResourceState.failed,
r2_status=const.ResourceState.deploying,
r2_last_non_deploying_status=const.ResourceState.skipped,
)
async def test_events_api_endpoints_events_across_versions(
server, client, environment, clienthelper, agent, resource_deployer):
"""
Ensure that events are captured across versions.
"""
# Version 1
version = await clienthelper.get_version()
rvid_r1_v1 = ResourceVersionIdStr(
f"std::File[agent1,path=/etc/file1],v={version}")
rvid_r2_v1 = ResourceVersionIdStr(
f"std::File[agent1,path=/etc/file2],v={version}")
resources = [
{
"path": "/etc/file1",
"id": rvid_r1_v1,
"requires": [rvid_r2_v1],
"purged": False,
"send_event": False
},
{
"path": "/etc/file2",
"id": rvid_r2_v1,
"requires": [],
"purged": False,
"send_event": False
},
]
await clienthelper.put_version_simple(resources, version)
# Deploy
await resource_deployer.deploy_resource(rvid=rvid_r2_v1)
# Version 2
version = await clienthelper.get_version()
rvid_r1_v2 = ResourceVersionIdStr(
f"std::File[agent1,path=/etc/file1],v={version}")
rvid_r2_v2 = ResourceVersionIdStr(
f"std::File[agent1,path=/etc/file2],v={version}")
rvid_r3_v2 = ResourceVersionIdStr(
f"std::File[agent1,path=/etc/file3],v={version}")
resources = [
{
"path": "/etc/file1",
"id": rvid_r1_v2,
"requires": [rvid_r2_v2, rvid_r3_v2],
"purged": False,
"send_event": False
},
{
"path": "/etc/file2",
"id": rvid_r2_v2,
"requires": [],
"purged": False,
"send_event": False
},
{
"path": "/etc/file3",
"id": rvid_r3_v2,
"requires": [],
"purged": False,
"send_event": False
},
]
await clienthelper.put_version_simple(resources, version)
# Deploy
await resource_deployer.deploy_resource(rvid=rvid_r2_v2)
await resource_deployer.deploy_resource(rvid=rvid_r3_v2)
# Version 3
version = await clienthelper.get_version()
rvid_r1_v3 = ResourceVersionIdStr(
f"std::File[agent1,path=/etc/file1],v={version}")
rid_v3_v3 = ResourceIdStr("std::File[agent1,path=/etc/file3]")
rvid_r3_v3 = ResourceVersionIdStr(f"{rid_v3_v3},v={version}")
resources = [
{
"path": "/etc/file1",
"id": rvid_r1_v3,
"requires": [rvid_r3_v3],
"purged": False,
"send_event": False
},
{
"path": "/etc/file3",
"id": rvid_r3_v3,
"requires": [],
"purged": False,
"send_event": False
},
]
await clienthelper.put_version_simple(resources, version)
# Deploy
await resource_deployer.deploy_resource(rvid=rvid_r3_v3,
status=const.ResourceState.failed)
action_id = await resource_deployer.start_deployment(rvid=rvid_r1_v3)
# Assert events
result = await agent._client.get_resource_events(tid=environment,
rvid=rvid_r1_v3)
assert result.code == 200
assert len(result.result["data"]) == 1
assert len(result.result["data"][rid_v3_v3]) == 2
assert result.result["data"][rid_v3_v3][0][
"action"] == const.ResourceAction.deploy
assert result.result["data"][rid_v3_v3][0][
"status"] == const.ResourceState.failed
assert result.result["data"][rid_v3_v3][1][
"action"] == const.ResourceAction.deploy
assert result.result["data"][rid_v3_v3][1][
"status"] == const.ResourceState.deployed
result = await agent._client.resource_did_dependency_change(
tid=environment, rvid=rvid_r1_v3)
assert result.code == 200
assert result.result["data"]
# Mark deployment r1 as done
await resource_deployer.deployment_finished(rvid=rvid_r1_v3,
action_id=action_id)
# Start new deployment for r1
await resource_deployer.start_deployment(rvid=rvid_r1_v3)
# Assert no move events
result = await agent._client.get_resource_events(tid=environment,
rvid=rvid_r1_v3)
assert result.code == 200
assert len(result.result["data"]) == 1
assert len(result.result["data"][rid_v3_v3]) == 0
result = await agent._client.resource_did_dependency_change(
tid=environment, rvid=rvid_r1_v3)
assert result.code == 200
assert not result.result["data"]