def test_project_unique(data_module):
project = data.Project(name="test")
yield project.insert()
project = data.Project(name="test")
with pytest.raises(pymongo.errors.DuplicateKeyError):
yield project.insert()
def test_config_model(data_module):
project = data.Project(name="test")
yield project.insert()
env = data.Environment(name="dev",
project=project.id,
repo_url="",
repo_branch="")
yield env.insert()
version = int(time.time())
cm = data.ConfigurationModel(environment=env.id,
version=version,
date=datetime.datetime.now(),
total=1,
version_info={})
yield cm.insert()
# create resources
key = "std::File[agent1,path=/etc/motd]"
res1 = data.Resource.new(environment=env.id,
resource_version_id=key + ",v=%d" % version,
attributes={"path": "/etc/motd"})
yield res1.insert()
agents = yield data.ConfigurationModel.get_agents(env.id, version)
assert len(agents) == 1
assert "agent1" in agents
async def test_resource_list_no_released_version(server, client):
"""Test that if there are no released versions of a resource, the result set is empty"""
project = data.Project(name="test")
await project.insert()
env = data.Environment(name="dev", project=project.id, repo_url="", repo_branch="")
await env.insert()
version = 1
cm = data.ConfigurationModel(
environment=env.id,
version=version,
date=datetime.now(),
total=1,
released=False,
version_info={},
)
await cm.insert()
path = f"/etc/file{1}"
key = f"std::File[agent1,path={path}]"
res1_v1 = data.Resource.new(
environment=env.id, resource_version_id=ResourceVersionIdStr(f"{key},v={version}"), attributes={"path": path}
)
await res1_v1.insert()
result = await client.resource_list(env.id)
assert result.code == 200
assert len(result.result["data"]) == 0
async def test_e2e_recompile_failure(compilerservice: CompilerService):
project = data.Project(name="test")
await project.insert()
env = data.Environment(name="dev",
project=project.id,
repo_url="",
repo_branch="")
await env.insert()
u1 = uuid.uuid4()
await compilerservice.request_recompile(
env, False, False, u1, env_vars={"my_unique_var": str(u1)})
u2 = uuid.uuid4()
await compilerservice.request_recompile(
env, False, False, u2, env_vars={"my_unique_var": str(u2)})
assert await compilerservice.is_compiling(env.id) == 200
async def compile_done():
res = await compilerservice.is_compiling(env.id)
print(res)
return res == 204
await retry_limited(compile_done, 10)
_, all_compiles = await compilerservice.get_reports(env)
all_reports = {
i["remote_id"]: await compilerservice.get_report(i["id"])
for i in all_compiles["reports"]
}
def assert_report(uid):
code, report = all_reports[uid]
report = report["report"]
assert report["remote_id"] == uid
assert not report["success"]
reports = report["reports"]
reports = {r["name"]: r for r in reports}
# stages
init = reports["Init"]
assert not init["errstream"]
assert "project found in" in init[
"outstream"] and "and no repository set" in init["outstream"]
# compile
comp = reports["Recompiling configuration model"]
assert "Unable to find an inmanta project (project.yml expected)" in comp[
"errstream"]
assert comp["returncode"] == 1
return report["requested"], report["started"], report["completed"]
r1, s1, f1 = assert_report(u1)
r2, s2, f2 = assert_report(u2)
assert r2 > r1
assert r1 < s1 < f1
assert r2 < s2 < f2
assert f1 < s2
def test_project(data_module):
project = data.Project(name="test")
yield project.insert()
projects = yield data.Project.get_list(name="test")
assert len(projects) == 1
assert projects[0].id == project.id
other = yield data.Project.get_by_id(project.id)
assert project != other
assert project.id == other.id
async def project_create(self, name: str,
project_id: Optional[uuid.UUID]) -> model.Project:
if project_id is None:
project_id = uuid.uuid4()
try:
project = data.Project(id=project_id, name=name)
await project.insert()
except asyncpg.exceptions.UniqueViolationError:
raise ServerError(f"A project with name {name} already exists.")
return project.to_dto()
def test_environment(data_module):
project = data.Project(name="test")
yield project.insert()
env = data.Environment(name="dev",
project=project.id,
repo_url="",
repo_branch="")
yield env.insert()
assert env.project == project.id
yield project.delete_cascade()
projects = yield data.Project.get_list()
envs = yield data.Environment.get_list()
assert len(projects) == 0
assert len(envs) == 0
def test_agent_process(data_module):
project = data.Project(name="test")
yield project.insert()
env = data.Environment(name="dev",
project=project.id,
repo_url="",
repo_branch="")
yield env.insert()
agent_proc = data.AgentProcess(hostname="testhost",
environment=env.id,
first_seen=datetime.datetime.now(),
last_seen=datetime.datetime.now(),
sid=uuid.uuid4())
yield agent_proc.insert()
agi1 = data.AgentInstance(process=agent_proc.id, name="agi1", tid=env.id)
yield agi1.insert()
agi2 = data.AgentInstance(process=agent_proc.id, name="agi2", tid=env.id)
yield agi2.insert()
agent = data.Agent(environment=env.id,
name="agi1",
last_failover=datetime.datetime.now(),
paused=False,
primary=agi1.id)
agent = yield agent.insert()
agents = yield data.Agent.get_list()
assert len(agents) == 1
agent = agents[0]
primary_instance = yield data.AgentInstance.get_by_id(agent.primary)
primary_process = yield data.AgentProcess.get_by_id(
primary_instance.process)
assert primary_process.id == agent_proc.id
async def test_pause_agent(server, cli):
project = data.Project(name="test")
await project.insert()
env1 = data.Environment(name="env1", project=project.id)
await env1.insert()
env2 = data.Environment(name="env2", project=project.id)
await env2.insert()
await data.Agent(environment=env1.id, name="agent1", paused=False).insert()
await data.Agent(environment=env1.id, name="agent2", paused=False).insert()
await data.Agent(environment=env2.id, name="agent3", paused=False).insert()
async def assert_agent_paused(env_id: uuid.UUID,
expected_records: Dict[str, bool]) -> None:
result = await cli.run("agent", "list", "-e", str(env_id))
assert result.exit_code == 0
output = result.stdout.replace(" ", "")
assert "Agent|Environment|Paused" in output
for (agent_name, paused) in expected_records.items():
assert f"{agent_name}|{env_id}|{paused}" in output
await assert_agent_paused(env_id=env1.id,
expected_records=dict(agent1=False,
agent2=False))
await assert_agent_paused(env_id=env2.id,
expected_records=dict(agent3=False))
# Pause
result = await cli.run("agent", "pause", "-e", str(env1.id), "--agent",
"agent1")
assert result.exit_code == 0
await assert_agent_paused(env_id=env1.id,
expected_records=dict(agent1=True, agent2=False))
await assert_agent_paused(env_id=env2.id,
expected_records=dict(agent3=False))
# Unpause
result = await cli.run("agent", "unpause", "-e", str(env1.id), "--agent",
"agent1")
assert result.exit_code == 0
await assert_agent_paused(env_id=env1.id,
expected_records=dict(agent1=False,
agent2=False))
await assert_agent_paused(env_id=env2.id,
expected_records=dict(agent3=False))
# Pause all agents in env1
result = await cli.run("agent", "pause", "-e", str(env1.id), "--all")
assert result.exit_code == 0
await assert_agent_paused(env_id=env1.id,
expected_records=dict(agent1=True, agent2=True))
await assert_agent_paused(env_id=env2.id,
expected_records=dict(agent3=False))
# Unpause all agents in env1
result = await cli.run("agent", "unpause", "-e", str(env1.id), "--all")
assert result.exit_code == 0
await assert_agent_paused(env_id=env1.id,
expected_records=dict(agent1=False,
agent2=False))
await assert_agent_paused(env_id=env2.id,
expected_records=dict(agent3=False))
# Mandatory option -e not specified
for action in ["pause", "unpause"]:
result = await cli.run("agent", action, "--agent", "agent1")
assert result.exit_code != 0
# --agent and --all are both set
for action in ["pause", "unpause"]:
result = await cli.run("agent", action, "-e", str(env1.id), "--agent",
"agent1", "--all")
assert result.exit_code != 0
# --agent and --all are both not set
for action in ["pause", "unpause"]:
result = await cli.run("agent", action, "-e", str(env1.id))
assert result.exit_code != 0
def __init__(self, dir: str) -> None:
self.src_dir: str = os.path.join(dir, "src")
self.project: data.Project = data.Project(name="test")
self._ready: bool = False
async def test_scheduler(server_config, init_dataclasses_and_load_schema,
caplog):
"""Test the scheduler part in isolation, mock out compile runner and listen to state updates"""
class Collector(CompileStateListener):
"""
Collect all state updates, optionally hang the processing of listeners
"""
def __init__(self):
self.seen = []
self.preseen = []
self.lock = Semaphore(1)
def reset(self):
self.seen = []
self.preseen = []
async def compile_done(self, compile: data.Compile):
self.preseen.append(compile)
print("Got compile done for ", compile.remote_id)
async with self.lock:
self.seen.append(compile)
async def hang(self):
await self.lock.acquire()
def release(self):
self.lock.release()
def verify(self, envs: uuid.UUID):
assert sorted([x.remote_id for x in self.seen]) == sorted(envs)
self.reset()
class HangRunner(object):
"""
compile runner mock, hang until released
"""
def __init__(self):
self.lock = Semaphore(0)
self.started = False
self.done = False
self.version = None
async def run(self, force_update: Optional[bool] = False):
self.started = True
await self.lock.acquire()
self.done = True
return True, None
def release(self):
self.lock.release()
class HookedCompilerService(CompilerService):
"""
hook in the hangrunner
"""
def __init__(self):
super(HookedCompilerService, self).__init__()
self.locks = {}
def _get_compile_runner(self, compile: data.Compile, project_dir: str):
print("Get Run: ", compile.remote_id, compile.id)
runner = HangRunner()
self.locks[compile.remote_id] = runner
return runner
def get_runner(self, remote_id: uuid.UUID) -> HangRunner:
return self.locks.get(remote_id)
# manual setup of server
server = Server()
cs = HookedCompilerService()
await cs.prestart(server)
await cs.start()
server.add_slice(cs)
notification_service = NotificationService()
await notification_service.prestart(server)
await notification_service.start()
collector = Collector()
cs.add_listener(collector)
async def request_compile(env: data.Environment) -> uuid.UUID:
"""Request compile for given env, return remote_id"""
u1 = uuid.uuid4()
# add unique environment variables to prevent merging in request_recompile
await cs.request_recompile(env,
False,
False,
u1,
env_vars={"uuid": str(u1)})
results = await data.Compile.get_by_remote_id(env.id, u1)
assert len(results) == 1
assert results[0].remote_id == u1
print("request: ", u1, results[0].id)
return u1
# setup projects in the database
project = data.Project(name="test")
await project.insert()
env1 = data.Environment(name="dev",
project=project.id,
repo_url="",
repo_branch="")
await env1.insert()
env2 = data.Environment(name="dev2",
project=project.id,
repo_url="",
repo_branch="")
await env2.insert()
# setup series of compiles for two envs
# e1 is for a plain run
# e2 is for server restart
e1 = [await request_compile(env1) for i in range(3)]
e2 = [await request_compile(env2) for i in range(4)]
print("env 1:", e1)
async def check_compile_in_sequence(env: data.Environment,
remote_ids: List[uuid.UUID], idx: int):
"""
Check integrity of a compile sequence and progress the hangrunner.
"""
before = remote_ids[:idx]
for rid in before:
prevrunner = cs.get_runner(rid)
assert prevrunner.done
if idx < len(remote_ids):
current = remote_ids[idx]
after = remote_ids[idx + 1:]
assert await cs.is_compiling(env.id) == 200
await retry_limited(lambda: cs.get_runner(current) is not None, 1)
await retry_limited(lambda: cs.get_runner(current).started, 1)
for rid in after:
nextrunner = cs.get_runner(rid)
assert nextrunner is None
cs.get_runner(current).release()
await asyncio.sleep(0)
await retry_limited(lambda: cs.get_runner(current).done, 1)
else:
async def isdone():
return await cs.is_compiling(env.id) == 204
await retry_limited(isdone, 1)
# run through env1, entire sequence
for i in range(4):
await check_compile_in_sequence(env1, e1, i)
collector.verify(e1)
print("env1 done")
print("env2 ", e2)
# make event collector hang
await collector.hang()
# progress two steps into env2
for i in range(2):
await check_compile_in_sequence(env2, e2, i)
assert not collector.seen
print(collector.preseen)
await retry_limited(lambda: len(collector.preseen) == 2, 1)
# test server restart
await notification_service.prestop()
await notification_service.stop()
await cs.prestop()
await cs.stop()
# in the log, find cancel of compile(hangs) and handler(hangs)
LogSequence(caplog, allow_errors=False).contains(
"inmanta.util", logging.WARNING,
"was cancelled").contains("inmanta.util", logging.WARNING,
"was cancelled").no_more_errors()
print("restarting")
# restart new server
cs = HookedCompilerService()
await cs.prestart(server)
await cs.start()
collector = Collector()
cs.add_listener(collector)
# complete the sequence, expect re-run of third compile
for i in range(3):
print(i)
await check_compile_in_sequence(env2, e2[2:], i)
# all are re-run, entire sequence present
collector.verify(e2)
await report_db_index_usage()
async def env_with_resources(server, client):
project = data.Project(name="test")
await project.insert()
env = data.Environment(name="dev",
project=project.id,
repo_url="",
repo_branch="")
await env.insert()
env2 = data.Environment(name="dev2",
project=project.id,
repo_url="",
repo_branch="")
await env2.insert()
env3 = data.Environment(name="dev3",
project=project.id,
repo_url="",
repo_branch="")
await env3.insert()
cm_times = []
for i in range(1, 10):
cm_times.append(
datetime.datetime.strptime(f"2021-07-07T1{i}:00:00.0",
"%Y-%m-%dT%H:%M:%S.%f"))
cm_time_idx = 0
resource_deploy_times = []
for i in range(30):
resource_deploy_times.append(
datetime.datetime.strptime(f"2021-07-07T11:{i}:00.0",
"%Y-%m-%dT%H:%M:%S.%f"))
# Add multiple versions of model, with 2 of them released
for i in range(1, 6):
cm = data.ConfigurationModel(
environment=env.id,
version=i,
date=cm_times[cm_time_idx],
total=1,
released=i != 1 and i != 5,
version_info={},
)
cm_time_idx += 1
await cm.insert()
cm = data.ConfigurationModel(
environment=env2.id,
version=4,
date=datetime.datetime.now(tz=datetime.timezone.utc),
total=1,
released=True,
version_info={},
)
cm_time_idx += 1
await cm.insert()
cm = data.ConfigurationModel(
environment=env3.id,
version=6,
date=datetime.datetime.now(tz=datetime.timezone.utc),
total=1,
released=True,
version_info={},
)
cm_time_idx += 1
await cm.insert()
resources = {
env.id: defaultdict(list),
env2.id: defaultdict(list),
env3.id: defaultdict(list)
}
def total_number_of_resources():
return sum([
len(resource_list_by_env) for resource_list_by_env in [[
specific_resource
for specific_resource_list in envdict.values()
for specific_resource in specific_resource_list
] for envdict in resources.values()]
])
async def create_resource(
path: str,
status: ResourceState,
version: int,
attributes: Dict[str, object],
agent: str = "internal",
resource_type: str = "std::File",
environment: UUID = env.id,
):
key = f"{resource_type}[{agent},path={path}]"
res = data.Resource.new(
environment=environment,
resource_version_id=ResourceVersionIdStr(f"{key},v={version}"),
attributes={
**attributes,
**{
"path": path
}
},
status=status,
last_deploy=resource_deploy_times[total_number_of_resources()],
)
await res.insert()
return res
# A resource with multiple resources in its requires list, and multiple versions where it was released,
# and is also present in versions that were not released
resources[env.id]["std::File[internal,path=/tmp/dir1/file1]"].append(
await create_resource(
"/tmp/dir1/file1",
ResourceState.undefined,
1,
{
"key1": "val1",
"requires": ["std::Directory[internal,path=/tmp/dir1],v=1"]
},
))
resources[env.id]["std::File[internal,path=/tmp/dir1/file1]"].append(
await create_resource(
"/tmp/dir1/file1",
ResourceState.skipped,
2,
{
"key1":
"modified_value",
"another_key":
"val",
"requires": [
"std::Directory[internal,path=/tmp/dir1],v=2",
"std::File[internal,path=/tmp/dir1/file2],v=2"
],
},
))
resources[env.id]["std::File[internal,path=/tmp/dir1/file1]"].append(
await create_resource(
"/tmp/dir1/file1",
ResourceState.deploying,
3,
{
"key1":
"modified_value",
"another_key":
"val",
"requires": [
"std::Directory[internal,path=/tmp/dir1],v=3",
"std::File[internal,path=/tmp/dir1/file2],v=3"
],
},
))
resources[env.id]["std::File[internal,path=/tmp/dir1/file1]"].append(
await create_resource(
"/tmp/dir1/file1",
ResourceState.deployed,
4,
{
"key1":
"modified_value",
"another_key":
"val",
"requires": [
"std::Directory[internal,path=/tmp/dir1],v=4",
"std::File[internal,path=/tmp/dir1/file2],v=4"
],
},
))
resources[env.id]["std::File[internal,path=/tmp/dir1/file1]"].append(
await create_resource(
"/tmp/dir1/file1",
ResourceState.undefined,
5,
{
"key1":
"modified_value",
"another_key":
"val",
"requires": [
"std::Directory[internal,path=/tmp/dir1],v=5",
"std::File[internal,path=/tmp/dir1/file2],v=5"
],
},
))
# A resource that didn't change its attributes, but was only released with the second version and has no requirements
resources[env.id]["std::Directory[internal,path=/tmp/dir1]"].append(
await create_resource(
"/tmp/dir1",
ResourceState.undefined,
1,
{
"key2": "val2",
"requires": []
},
resource_type="std::Directory",
))
resources[env.id]["std::Directory[internal,path=/tmp/dir1]"].append(
await create_resource("/tmp/dir1",
ResourceState.deploying,
2, {
"key2": "val2",
"requires": []
},
resource_type="std::Directory"))
resources[env.id]["std::Directory[internal,path=/tmp/dir1]"].append(
await create_resource("/tmp/dir1",
ResourceState.deployed,
3, {
"key2": "val2",
"requires": []
},
resource_type="std::Directory"))
resources[env.id]["std::Directory[internal,path=/tmp/dir1]"].append(
await create_resource("/tmp/dir1",
ResourceState.deployed,
4, {
"key2": "val2",
"requires": []
},
resource_type="std::Directory"))
# A resource that changed the attributes in the last released version,
# so the last and the first time the attributes are the same, is the same as well;
# And it also has a single requirement
resources[env.id]["std::File[internal,path=/tmp/dir1/file2]"].append(
await create_resource("/tmp/dir1/file2", ResourceState.undefined, 1, {
"key3": "val3",
"requires": []
}))
resources[env.id]["std::File[internal,path=/tmp/dir1/file2]"].append(
await create_resource(
"/tmp/dir1/file2",
ResourceState.deployed,
2,
{
"key3": "val3",
"requires": ["std::Directory[internal,path=/tmp/dir1],v=2"]
},
))
resources[env.id]["std::File[internal,path=/tmp/dir1/file2]"].append(
await create_resource(
"/tmp/dir1/file2",
ResourceState.deployed,
3,
{
"key3": "val3",
"requires": ["std::Directory[internal,path=/tmp/dir1],v=3"]
},
))
resources[env.id]["std::File[internal,path=/tmp/dir1/file2]"].append(
await create_resource(
"/tmp/dir1/file2",
ResourceState.deploying,
4,
{
"key3": "val3updated",
"requires": ["std::Directory[internal,path=/tmp/dir1],v=4"]
},
))
# Add an unreleased resource
resources[env.id]["std::File[internal,path=/etc/filexyz]"].append(
await create_resource(
"/etc/filexyz",
ResourceState.undefined,
5,
{
"key4": "val4",
"requires": []
},
))
resources[env.id]["std::File[internal,path=/etc/never_deployed]"].append(
await create_resource(
"/etc/never_deployed",
ResourceState.undefined,
3,
{
"key5": "val5",
"requires": []
},
))
resources[env.id]["std::File[internal,path=/etc/never_deployed]"].append(
await create_resource(
"/etc/never_deployed",
ResourceState.unavailable,
4,
{
"key5": "val5",
"requires": []
},
))
resources[env.id][
"std::File[internal,path=/etc/deployed_only_with_different_hash]"].append(
await create_resource(
"/etc/deployed_only_with_different_hash",
ResourceState.deployed,
3,
{
"key6": "val6",
"requires": []
},
))
resources[env.id][
"std::File[internal,path=/etc/deployed_only_with_different_hash]"].append(
await create_resource(
"/etc/deployed_only_with_different_hash",
ResourceState.undefined,
4,
{
"key6": "val6different",
"requires": []
},
))
resources[env.id][
"std::File[internal,path=/etc/deployed_only_in_earlier_version]"].append(
await create_resource(
"/etc/deployed_only_in_earlier_version",
ResourceState.deployed,
3,
{
"key7":
"val7",
"requires": [
"std::File[internal,path=/etc/requirement_in_later_version],v=3"
]
},
))
resources[env.id][
"std::File[internal,path=/etc/requirement_in_later_version]"].append(
await create_resource(
"/etc/requirement_in_later_version",
ResourceState.deploying,
3,
{
"key8": "val8",
"requires": []
},
))
resources[env.id][
"std::File[internal,path=/etc/requirement_in_later_version]"].append(
await create_resource(
"/etc/requirement_in_later_version",
ResourceState.deployed,
4,
{
"key8": "val8",
"requires": []
},
))
resources[env.id][
"std::File[internal,path=/etc/requirement_in_later_version]"].append(
await create_resource(
"/etc/requirement_in_later_version",
ResourceState.skipped,
5,
{
"key8": "val8",
"requires": []
},
))
resources[env.id]["std::File[internal,path=/tmp/orphaned]"].append(
await create_resource(
"/tmp/orphaned",
ResourceState.deployed,
3,
{
"key9": "val9",
"requires": ["std::File[internal,path=/tmp/orphaned_req],v=3"]
},
))
resources[env.id]["std::File[internal,path=/tmp/orphaned_req]"].append(
await create_resource(
"/tmp/orphaned_req",
ResourceState.deployed,
3,
{
"key9": "val9",
"requires": []
},
))
# Add the same resources the first one requires in another environment
resources[env2.id]["std::File[internal,path=/tmp/dir1/file2]"].append(
await create_resource(
"/tmp/dir1/file2",
ResourceState.unavailable,
4,
{
"key3": "val3",
"requires": ["std::Directory[internal,path=/tmp/dir1],v=4"]
},
resource_type="std::Directory",
environment=env2.id,
))
resources[env2.id]["std::Directory[internal,path=/tmp/dir1]"].append(
await create_resource(
"/tmp/dir1",
ResourceState.available,
4,
{
"key2": "val2",
"requires": []
},
resource_type="std::Directory",
environment=env2.id,
))
# Add the same main resource to another environment with higher version
resources[env3.id]["std::File[internal,path=/tmp/dir1/file1]"].append(
await create_resource(
"/tmp/dir1/file1",
ResourceState.deploying,
6,
{
"key1":
"modified_value",
"another_key":
"val",
"requires": [
"std::Directory[internal,path=/tmp/dir1],v=6",
"std::File[internal,path=/tmp/dir1/file2],v=6"
],
},
environment=env3.id,
))
ids = {
"multiple_requires": "std::File[internal,path=/tmp/dir1/file1]",
"no_requires": "std::Directory[internal,path=/tmp/dir1]",
"single_requires": "std::File[internal,path=/tmp/dir1/file2]",
"unreleased": "std::File[internal,path=/etc/filexyz]",
"never_deployed": "std::File[internal,path=/etc/never_deployed]",
"deployed_only_with_different_hash":
"std::File[internal,path=/etc/deployed_only_with_different_hash]",
"deployed_only_in_earlier_version":
"std::File[internal,path=/etc/deployed_only_in_earlier_version]",
"orphaned_and_requires_orphaned":
"std::File[internal,path=/tmp/orphaned]",
}
yield env, cm_times, ids, resources
async def environments_with_versions(
server,
client) -> Tuple[Dict[str, uuid.UUID], List[datetime.datetime]]:
project = data.Project(name="test")
await project.insert()
env = data.Environment(name="dev",
project=project.id,
repo_url="",
repo_branch="")
await env.insert()
cm_timestamps = []
for i in range(0, 10):
cm_timestamps.append(
datetime.datetime.strptime(f"2021-12-06T11:{i}:00.0",
"%Y-%m-%dT%H:%M:%S.%f"))
# Add multiple versions of model
# 1: skipped_candidate, 2,3: retired, 4,5,6: skipped_candidate, 7: active, 8,9: candidate
for i in range(1, 10):
cm = data.ConfigurationModel(
environment=env.id,
version=i,
date=cm_timestamps[i - 1],
total=1,
released=i in {2, 3, 7},
version_info={
"export_metadata": {
"message": "Recompile model because state transition",
"type": "lsm_export"
}
} if i % 2 else {
"export_metadata": {
"message":
"Recompile model because one or more parameters were updated",
"type": "param"
}
},
)
await cm.insert()
env2 = data.Environment(name="dev-test2",
project=project.id,
repo_url="",
repo_branch="")
await env2.insert()
cm = data.ConfigurationModel(
environment=env2.id,
version=11,
date=datetime.datetime.now(),
total=1,
released=True,
version_info={},
)
await cm.insert()
env3 = data.Environment(name="dev-test3",
project=project.id,
repo_url="",
repo_branch="")
await env3.insert()
cm = data.ConfigurationModel(
environment=env3.id,
version=7,
date=datetime.datetime.now(),
total=1,
released=False,
version_info={},
)
await cm.insert()
env4 = data.Environment(name="dev-test4",
project=project.id,
repo_url="",
repo_branch="")
await env4.insert()
environments = {
"multiple_versions": env.id,
"single_released_version": env2.id,
"no_released_version": env3.id,
"no_versions": env4.id,
}
yield environments, cm_timestamps
async def test_resource_action_pagination(postgresql_client, client,
clienthelper, server, agent):
""" Test querying resource actions via the API, including the pagination links."""
project = data.Project(name="test")
await project.insert()
env = data.Environment(name="dev",
project=project.id,
repo_url="",
repo_branch="")
await env.insert()
# Add multiple versions of model
for i in range(0, 11):
cm = data.ConfigurationModel(
environment=env.id,
version=i,
date=datetime.now(),
total=1,
version_info={},
)
await cm.insert()
# Add resource actions for motd
motd_first_start_time = datetime.now()
earliest_action_id = uuid.uuid4()
resource_action = data.ResourceAction(
environment=env.id,
version=0,
resource_version_ids=[f"std::File[agent1,path=/etc/motd],v={0}"],
action_id=earliest_action_id,
action=const.ResourceAction.deploy,
started=motd_first_start_time - timedelta(minutes=1),
)
await resource_action.insert()
resource_action.add_logs([
data.LogLine.log(logging.INFO,
"Successfully stored version %(version)d",
version=0)
])
await resource_action.save()
action_ids_with_the_same_timestamp = []
for i in range(1, 6):
action_id = uuid.uuid4()
action_ids_with_the_same_timestamp.append(action_id)
resource_action = data.ResourceAction(
environment=env.id,
version=i,
resource_version_ids=[f"std::File[agent1,path=/etc/motd],v={i}"],
action_id=action_id,
action=const.ResourceAction.deploy,
started=motd_first_start_time,
)
await resource_action.insert()
resource_action.add_logs([
data.LogLine.log(logging.INFO,
"Successfully stored version %(version)d",
version=i)
])
await resource_action.save()
action_ids_with_the_same_timestamp = sorted(
action_ids_with_the_same_timestamp, reverse=True)
later_action_id = uuid.uuid4()
resource_action = data.ResourceAction(
environment=env.id,
version=6,
resource_version_ids=[f"std::File[agent1,path=/etc/motd],v={6}"],
action_id=later_action_id,
action=const.ResourceAction.deploy,
started=motd_first_start_time + timedelta(minutes=6),
)
await resource_action.insert()
resource_action.add_logs([
data.LogLine.log(logging.INFO,
"Successfully stored version %(version)d",
version=6)
])
await resource_action.save()
for i in range(0, 11):
res1 = data.Resource.new(
environment=env.id,
resource_version_id="std::File[agent1,path=/etc/motd],v=%s" %
str(i),
status=const.ResourceState.deployed,
last_deploy=datetime.now() + timedelta(minutes=i),
attributes={
"attr": [{
"a": 1,
"b": "c"
}],
"path": "/etc/motd"
},
)
await res1.insert()
result = await client.get_resource_actions(
tid=env.id,
resource_type="std::File",
attribute="path",
attribute_value="/etc/motd",
last_timestamp=motd_first_start_time + timedelta(minutes=7),
limit=2,
)
assert result.code == 200
resource_actions = result.result["data"]
expected_action_ids = [later_action_id
] + action_ids_with_the_same_timestamp[:1]
assert [
uuid.UUID(resource_action["action_id"])
for resource_action in resource_actions
] == expected_action_ids
# Use the next link for pagination
next_page = result.result["links"]["next"]
port = opt.get_bind_port()
base_url = "http://localhost:%s" % (port, )
url = f"{base_url}{next_page}"
client = AsyncHTTPClient()
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(env.id)},
)
response = await client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
second_page_action_ids = [
uuid.UUID(resource_action["action_id"])
for resource_action in response["data"]
]
assert second_page_action_ids == action_ids_with_the_same_timestamp[1:3]
next_page = response["links"]["next"]
url = f"{base_url}{next_page}"
request.url = url
response = await client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
third_page_action_ids = [
uuid.UUID(resource_action["action_id"])
for resource_action in response["data"]
]
assert third_page_action_ids == action_ids_with_the_same_timestamp[3:5]
# Go back to the previous page
prev_page = response["links"]["prev"]
url = f"{base_url}{prev_page}"
request.url = url
response = await client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
action_ids = [
uuid.UUID(resource_action["action_id"])
for resource_action in response["data"]
]
assert action_ids == second_page_action_ids
# And back to the third
prev_page = response["links"]["next"]
url = f"{base_url}{prev_page}"
request.url = url
response = await client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
action_ids = [
uuid.UUID(resource_action["action_id"])
for resource_action in response["data"]
]
assert action_ids == third_page_action_ids
async def test_has_only_one_version_from_resource(server, client):
"""Test querying resources, when there are multiple released versions of a resource.
The query should return only the latest one from those
"""
project = data.Project(name="test")
await project.insert()
env = data.Environment(name="dev", project=project.id, repo_url="", repo_branch="")
await env.insert()
# Add multiple versions of model, with 2 of them released
for i in range(1, 5):
cm = data.ConfigurationModel(
environment=env.id,
version=i,
date=datetime.now(),
total=1,
released=i != 1 and i != 4,
version_info={},
)
await cm.insert()
version = 1
path = "/etc/file" + str(1)
key = "std::File[agent1,path=" + path + "]"
res1_v1 = data.Resource.new(environment=env.id, resource_version_id=key + ",v=%d" % version, attributes={"path": path})
await res1_v1.insert()
version = 2
res1_v2 = data.Resource.new(
environment=env.id,
resource_version_id=key + ",v=%d" % version,
attributes={"path": path},
status=ResourceState.deploying,
)
await res1_v2.insert()
version = 3
res1_v3 = data.Resource.new(
environment=env.id,
resource_version_id=key + ",v=%d" % version,
attributes={"path": path},
status=ResourceState.deployed,
)
await res1_v3.insert()
version = 4
res1_v4 = data.Resource.new(
environment=env.id,
resource_version_id=key + ",v=%d" % version,
attributes={"path": path, "new_attr": 123, "requires": ["abc"]},
status=ResourceState.deployed,
)
await res1_v4.insert()
version = 1
path = "/etc/file" + str(2)
key = "std::File[agent1,path=" + path + "]"
res2_v1 = data.Resource.new(environment=env.id, resource_version_id=key + ",v=%d" % version, attributes={"path": path})
await res2_v1.insert()
version = 2
res2_v2 = data.Resource.new(
environment=env.id,
resource_version_id=key + ",v=%d" % version,
attributes={"path": path},
status=ResourceState.deploying,
)
await res2_v2.insert()
result = await client.resource_list(env.id, sort="status.asc")
assert result.code == 200
assert len(result.result["data"]) == 2
assert result.result["data"][0]["status"] == "deployed"
assert result.result["data"][0]["requires"] == []
# Orphaned, since there is already a version 3 released
assert result.result["data"][1]["status"] == "orphaned"
async def test_notification_cleanup_on_start(init_dataclasses_and_load_schema,
async_finalizer,
server_config) -> None:
project = data.Project(name="test")
await project.insert()
env_with_default_retention = data.Environment(name="testenv",
project=project.id)
await env_with_default_retention.insert()
env_with_short_retention = data.Environment(name="testenv2",
project=project.id)
await env_with_short_retention.insert()
await env_with_short_retention.set(data.NOTIFICATION_RETENTION, 30)
timestamps = [
datetime.datetime.now().astimezone() - datetime.timedelta(days=366),
datetime.datetime.now().astimezone() - datetime.timedelta(days=35),
datetime.datetime.now().astimezone(),
]
async def insert_notifications_with_timestamps(
timestamps: Sequence[datetime.datetime],
environment_ids: Sequence[uuid.UUID]) -> None:
for env_id in environment_ids:
for created in timestamps:
await data.Notification(
title="Notification",
message="Something happened",
environment=env_id,
severity=const.NotificationSeverity.message,
uri="/api/v2/notification",
created=created,
read=False,
cleared=False,
).insert()
await insert_notifications_with_timestamps(
timestamps,
[env_with_default_retention.id, env_with_short_retention.id])
server = Server()
notification_service = NotificationService()
compiler_service = CompilerService()
server.add_slice(compiler_service)
server.add_slice(notification_service)
await server.start()
async_finalizer.add(server.stop)
async def notification_cleaned_up(
env_id: uuid.UUID, expected_length_after_cleanup: int) -> bool:
default_env_notifications = await data.Notification.get_list(
environment=env_id)
return len(default_env_notifications) == expected_length_after_cleanup
await retry_limited(partial(notification_cleaned_up,
env_with_default_retention.id, 2),
timeout=10)
default_env_notifications = await data.Notification.get_list(
environment=env_with_default_retention.id,
order="DESC",
order_by_column="created")
# Only the oldest one is deleted
assert len(default_env_notifications) == 2
assert default_env_notifications[0].created == timestamps[2]
assert default_env_notifications[1].created == timestamps[1]
await retry_limited(partial(notification_cleaned_up,
env_with_short_retention.id, 1),
timeout=10)
short_retention_notifications = await data.Notification.get_list(
environment=env_with_short_retention.id,
order="DESC",
order_by_column="created")
# Only the latest one is kept
assert len(short_retention_notifications) == 1
assert short_retention_notifications[0].created == timestamps[2]
async def env_with_resources(server, client):
project = data.Project(name="test")
await project.insert()
env = data.Environment(name="dev",
project=project.id,
repo_url="",
repo_branch="")
await env.insert()
# Add multiple versions of model, with 2 of them released
for i in range(1, 4):
cm = data.ConfigurationModel(
environment=env.id,
version=i,
date=datetime.now(),
total=1,
released=i != 1,
version_info={},
)
await cm.insert()
async def create_resource(agent: str,
path: str,
resource_type: str,
versions: List[int],
environment: UUID = env.id):
for version in versions:
key = f"{resource_type}[{agent},path={path}]"
res = data.Resource.new(
environment=environment,
resource_version_id=ResourceVersionIdStr(f"{key},v={version}"),
attributes={
"path": path,
"v": version
},
status=ResourceState.deployed,
)
await res.insert()
await create_resource("agent1", "/etc/file1", "std::File", [1, 2, 3])
await create_resource("agent1", "/etc/file2", "std::File", [1, 2])
await create_resource("agent2", "/etc/file3", "std::File", [2])
await create_resource("agent2", "/tmp/file4", "std::File", [3])
await create_resource("agent2", "/tmp/dir5", "std::Directory", [3])
await create_resource("agent2", "/tmp/dir6", "std::Directory", [3])
await create_resource("agent2", "/tmp/dir7", "std::Directory", [3])
await create_resource("agent3", "/tmp/dir8", "std::Directory", [3])
env2 = data.Environment(name="dev-test2",
project=project.id,
repo_url="",
repo_branch="")
await env2.insert()
cm = data.ConfigurationModel(
environment=env2.id,
version=3,
date=datetime.now(),
total=1,
released=True,
version_info={},
)
await cm.insert()
await create_resource("agent1",
"/tmp/file7",
"std::File", [3],
environment=env2.id)
await create_resource("agent1",
"/tmp/file2",
"std::File", [3],
environment=env2.id)
yield env
