async def get_agent(server, environment, *endpoints, hostname="nodes1"): agentmanager = server.get_slice(SLICE_AGENT_MANAGER) prelen = len(agentmanager.sessions) agent = Agent( hostname=hostname, environment=environment, agent_map={agent: "localhost" for agent in endpoints}, code_loader=False ) for agentname in endpoints: await agent.add_end_point_name(agentname) await agent.start() await retry_limited(lambda: len(agentmanager.sessions) == prelen + 1, 10) return agent
async def agent(server, environment): agentmanager = server.get_slice(SLICE_AGENT_MANAGER) config.Config.set("config", "agent-deploy-interval", "0") config.Config.set("config", "agent-repair-interval", "0") a = Agent(hostname="node1", environment=environment, agent_map={"agent1": "localhost"}, code_loader=False) await a.add_end_point_name("agent1") await a.start() await utils.retry_limited(lambda: len(agentmanager.sessions) == 1, 10) yield a await a.stop()
async def create_agent( environment: uuid.UUID, hostname: Optional[str] = None, agent_map: Optional[Dict[str, str]] = None, code_loader: bool = False, agent_names: List[str] = [], ) -> None: a = Agent(hostname=hostname, environment=environment, agent_map=agent_map, code_loader=code_loader) for agent_name in agent_names: await a.add_end_point_name(agent_name) await a.start() started_agents.append(a) await utils.retry_limited(lambda: a.sessionid in agentmanager.sessions, 10) return a
async def setup_agent(self, server, environment): agentmanager = server.get_slice(SLICE_AGENT_MANAGER) endpoints = list(self.results.keys()) config.Config.set("config", "agent-deploy-interval", "0") config.Config.set("config", "agent-repair-interval", "0") a = Agent(hostname="node1", environment=environment, agent_map={e: "localhost" for e in endpoints}, code_loader=False) for e in endpoints: await a.add_end_point_name(e) await a.start() await utils.retry_limited(lambda: len(agentmanager.sessions) == 1, 10) return a
def test_purge_on_delete_ignore(io_loop, client, server, environment): """ Test purge on delete behavior for resources that have not longer purged_on_delete set """ agent = Agent(io_loop, "localhost", {"blah": "localhost"}, environment=environment) agent.start() aclient = agent._client # Version 1 with purge_on_delete true version = 1 resources = [{ 'group': 'root', 'hash': '89bf880a0dc5ffc1156c8d958b4960971370ee6a', 'id': 'std::File[vm1,path=/tmp/file1],v=%d' % version, 'owner': 'root', 'path': '/tmp/file1', 'permissions': 644, 'purged': False, 'reload': False, 'requires': [], 'purge_on_delete': True, 'version': version }] res = yield client.put_version(tid=environment, version=version, resources=resources, unknowns=[], version_info={}) assert res.code == 200 # Release the model and set all resources as deployed result = yield client.release_version(environment, version, push=False) assert result.code == 200 now = datetime.now() result = yield aclient.resource_action_update( environment, ['std::File[vm1,path=/tmp/file1],v=%d' % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = yield client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["version"] == version assert result.result["model"]["total"] == len(resources) assert result.result["model"]["done"] == len(resources) assert result.result["model"]["released"] assert result.result["model"]["result"] == const.VersionState.success.name # Version 2 with purge_on_delete false version = 2 resources = [{ 'group': 'root', 'hash': '89bf880a0dc5ffc1156c8d958b4960971370ee6a', 'id': 'std::File[vm1,path=/tmp/file1],v=%d' % version, 'owner': 'root', 'path': '/tmp/file1', 'permissions': 644, 'purged': False, 'reload': False, 'requires': [], 'purge_on_delete': False, 'version': version }] res = yield client.put_version(tid=environment, version=version, resources=resources, unknowns=[], version_info={}) assert res.code == 200 # Release the model and set all resources as deployed result = yield client.release_version(environment, version, push=False) assert result.code == 200 now = datetime.now() result = yield aclient.resource_action_update( environment, ['std::File[vm1,path=/tmp/file1],v=%d' % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = yield client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["version"] == version assert result.result["model"]["total"] == len(resources) assert result.result["model"]["done"] == len(resources) assert result.result["model"]["released"] assert result.result["model"]["result"] == const.VersionState.success.name # Version 3 with no resources version = 3 resources = [] res = yield client.put_version(tid=environment, version=version, resources=resources, unknowns=[], version_info={}) assert res.code == 200 result = yield client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["version"] == version assert result.result["model"]["total"] == len(resources)
def test_resource_update(io_loop, client, server, environment): """ Test updating resources and logging """ agent = Agent(io_loop, "localhost", {"blah": "localhost"}, environment=environment) agent.start() aclient = agent._client version = int(time.time()) resources = [] for j in range(10): resources.append({ 'group': 'root', 'hash': '89bf880a0dc5ffc1156c8d958b4960971370ee6a', 'id': 'std::File[vm1,path=/tmp/file%d],v=%d' % (j, version), 'owner': 'root', 'path': '/tmp/file%d' % j, 'permissions': 644, 'purged': False, 'reload': False, 'requires': [], 'version': version }) res = yield client.put_version(tid=environment, version=version, resources=resources, unknowns=[], version_info={}) assert (res.code == 200) result = yield client.release_version(environment, version, push=False) assert result.code == 200 resource_ids = [x["id"] for x in resources] # Start the deploy action_id = uuid.uuid4() now = datetime.now() result = yield aclient.resource_action_update(environment, resource_ids, action_id, "deploy", now) assert (result.code == 200) # Get the status from a resource result = yield client.get_resource(tid=environment, id=resource_ids[0], logs=True) assert (result.code == 200) logs = {x["action"]: x for x in result.result["logs"]} assert ("deploy" in logs) assert ("finished" not in logs["deploy"]) assert ("messages" not in logs["deploy"]) assert ("changes" not in logs["deploy"]) # Send some logs result = yield aclient.resource_action_update( environment, resource_ids, action_id, "deploy", messages=[ data.LogLine.log(const.LogLevel.INFO, "Test log %(a)s %(b)s", a="a", b="b") ]) assert (result.code == 200) # Get the status from a resource result = yield client.get_resource(tid=environment, id=resource_ids[0], logs=True) assert (result.code == 200) logs = {x["action"]: x for x in result.result["logs"]} assert ("deploy" in logs) assert ("messages" in logs["deploy"]) assert (len(logs["deploy"]["messages"]) == 1) assert (logs["deploy"]["messages"][0]["msg"] == "Test log a b") assert ("finished" not in logs["deploy"]) assert ("changes" not in logs["deploy"]) # Finish the deploy now = datetime.now() changes = { x: { "owner": { "old": "root", "current": "inmanta" } } for x in resource_ids } result = yield aclient.resource_action_update(environment, resource_ids, action_id, "deploy", finished=now, changes=changes) assert (result.code == 500) result = yield aclient.resource_action_update(environment, resource_ids, action_id, "deploy", status="deployed", finished=now, changes=changes) assert (result.code == 200) result = yield client.get_version(environment, version) assert (result.code == 200) assert result.result["model"]["done"] == 10
def test_purge_on_delete(io_loop, client, server, environment): """ Test purge on delete of resources """ agent = Agent(io_loop, "localhost", {"blah": "localhost"}, environment=environment) agent.start() aclient = agent._client version = 1 resources = [{ 'group': 'root', 'hash': '89bf880a0dc5ffc1156c8d958b4960971370ee6a', 'id': 'std::File[vm1,path=/tmp/file1],v=%d' % version, 'owner': 'root', 'path': '/tmp/file1', 'permissions': 644, 'purged': False, 'reload': False, 'requires': [], 'purge_on_delete': True, 'version': version }, { 'group': 'root', 'hash': 'b4350bef50c3ec3ee532d4a3f9d6daedec3d2aba', 'id': 'std::File[vm1,path=/tmp/file2],v=%d' % version, 'owner': 'root', 'path': '/tmp/file2', 'permissions': 644, 'purged': False, 'reload': False, 'purge_on_delete': True, 'requires': ['std::File[vm1,path=/tmp/file1],v=%d' % version], 'version': version }, { 'group': 'root', 'hash': '89bf880a0dc5ffc1156c8d958b4960971370ee6a', 'id': 'std::File[vm1,path=/tmp/file3],v=%d' % version, 'owner': 'root', 'path': '/tmp/file3', 'permissions': 644, 'purged': False, 'reload': False, 'requires': [], 'purge_on_delete': True, 'version': version }] res = yield client.put_version(tid=environment, version=version, resources=resources, unknowns=[], version_info={}) assert res.code == 200 # Release the model and set all resources as deployed result = yield client.release_version(environment, version, push=False) assert result.code == 200 now = datetime.now() result = yield aclient.resource_action_update( environment, ['std::File[vm1,path=/tmp/file1],v=%d' % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = yield aclient.resource_action_update( environment, ['std::File[vm1,path=/tmp/file2],v=%d' % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = yield aclient.resource_action_update( environment, ['std::File[vm1,path=/tmp/file3],v=%d' % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = yield client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["version"] == version assert result.result["model"]["total"] == len(resources) assert result.result["model"]["done"] == len(resources) assert result.result["model"]["released"] assert result.result["model"]["result"] == const.VersionState.success.name # New version with only file3 version = 2 res3 = { 'group': 'root', 'hash': '89bf880a0dc5ffc1156c8d958b4960971370ee6a', 'id': 'std::File[vm1,path=/tmp/file3],v=%d' % version, 'owner': 'root', 'path': '/tmp/file3', 'permissions': 644, 'purged': False, 'reload': False, 'requires': [], 'purge_on_delete': True, 'version': version } res = yield client.put_version(tid=environment, version=version, resources=[res3], unknowns=[], version_info={}) assert result.code == 200 result = yield client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["total"] == 3 # validate requires and provides file1 = [x for x in result.result["resources"] if "file1" in x["id"]][0] file2 = [x for x in result.result["resources"] if "file2" in x["id"]][0] file3 = [x for x in result.result["resources"] if "file3" in x["id"]][0] assert file1["attributes"]["purged"] assert file2["attributes"]["purged"] assert not file3["attributes"]["purged"]
async def test_dryrun_and_deploy(server, client, resource_container, environment): """ dryrun and deploy a configuration model There is a second agent with an undefined resource. The server will shortcut the dryrun and deploy for this resource without an agent being present. """ agentmanager = server.get_slice(SLICE_AGENT_MANAGER) agent = Agent(hostname="node1", environment=environment, agent_map={"agent1": "localhost"}, code_loader=False) await agent.add_end_point_name("agent1") await agent.start() await retry_limited(lambda: len(agentmanager.sessions) == 1, 10) resource_container.Provider.set("agent1", "key2", "incorrect_value") resource_container.Provider.set("agent1", "key3", "value") clienthelper = ClientHelper(client, environment) version = await clienthelper.get_version() resources = [ { "key": "key1", "value": "value1", "id": "test::Resource[agent1,key=key1],v=%d" % version, "send_event": False, "purged": False, "requires": ["test::Resource[agent1,key=key2],v=%d" % version], }, { "key": "key2", "value": "value2", "id": "test::Resource[agent1,key=key2],v=%d" % version, "send_event": False, "requires": [], "purged": False, }, { "key": "key3", "value": None, "id": "test::Resource[agent1,key=key3],v=%d" % version, "send_event": False, "requires": [], "purged": True, }, { "key": "key4", "value": execute.util.Unknown(source=None), "id": "test::Resource[agent2,key=key4],v=%d" % version, "send_event": False, "requires": [], "purged": False, }, { "key": "key5", "value": "val", "id": "test::Resource[agent2,key=key5],v=%d" % version, "send_event": False, "requires": ["test::Resource[agent2,key=key4],v=%d" % version], "purged": False, }, { "key": "key6", "value": "val", "id": "test::Resource[agent2,key=key6],v=%d" % version, "send_event": False, "requires": ["test::Resource[agent2,key=key5],v=%d" % version], "purged": False, }, ] status = {"test::Resource[agent2,key=key4]": const.ResourceState.undefined} result = await client.put_version( tid=environment, version=version, resources=resources, resource_state=status, unknowns=[], version_info={}, compiler_version=get_compiler_version(), ) assert result.code == 200 mod_db = await data.ConfigurationModel.get_version(uuid.UUID(environment), version) undep = await mod_db.get_undeployable() assert undep == ["test::Resource[agent2,key=key4]"] undep = await mod_db.get_skipped_for_undeployable() assert undep == [ "test::Resource[agent2,key=key5]", "test::Resource[agent2,key=key6]" ] # request a dryrun result = await client.dryrun_request(environment, version) assert result.code == 200 assert result.result["dryrun"]["total"] == len(resources) assert result.result["dryrun"]["todo"] == len(resources) # get the dryrun results result = await client.dryrun_list(environment, version) assert result.code == 200 assert len(result.result["dryruns"]) == 1 while result.result["dryruns"][0]["todo"] > 0: result = await client.dryrun_list(environment, version) await asyncio.sleep(0.1) dry_run_id = result.result["dryruns"][0]["id"] result = await client.dryrun_report(environment, dry_run_id) assert result.code == 200 changes = result.result["dryrun"]["resources"] assert changes[resources[0]["id"]]["changes"]["purged"]["current"] assert not changes[resources[0]["id"]]["changes"]["purged"]["desired"] assert changes[resources[0]["id"]]["changes"]["value"]["current"] is None assert changes[resources[0]["id"]]["changes"]["value"][ "desired"] == resources[0]["value"] assert changes[resources[1] ["id"]]["changes"]["value"]["current"] == "incorrect_value" assert changes[resources[1]["id"]]["changes"]["value"][ "desired"] == resources[1]["value"] assert not changes[resources[2]["id"]]["changes"]["purged"]["current"] assert changes[resources[2]["id"]]["changes"]["purged"]["desired"] # do a deploy result = await client.release_version( environment, version, True, const.AgentTriggerMethod.push_full_deploy) assert result.code == 200 assert not result.result["model"]["deployed"] assert result.result["model"]["released"] assert result.result["model"]["total"] == 6 assert result.result["model"]["result"] == "deploying" result = await client.get_version(environment, version) assert result.code == 200 await _wait_until_deployment_finishes(client, environment, version) result = await client.get_version(environment, version) assert result.result["model"]["done"] == len(resources) assert resource_container.Provider.isset("agent1", "key1") assert resource_container.Provider.get("agent1", "key1") == "value1" assert resource_container.Provider.get("agent1", "key2") == "value2" assert not resource_container.Provider.isset("agent1", "key3") actions = await data.ResourceAction.get_list() assert sum([ len(x.resource_version_ids) for x in actions if x.status == const.ResourceState.undefined ]) == 1 assert sum([ len(x.resource_version_ids) for x in actions if x.status == const.ResourceState.skipped_for_undefined ]) == 2 await agent.stop()
def test_get_resource_for_agent(io_loop, motor, server_multi, client, environment): """ Test the server to manage the updates on a model during agent deploy """ agent = Agent(io_loop, "localhost", {"nvblah": "localhost"}, environment=environment) agent.start() aclient = agent._client version = 1 resources = [{ 'group': 'root', 'hash': '89bf880a0dc5ffc1156c8d958b4960971370ee6a', 'id': 'std::File[vm1.dev.inmanta.com,path=/etc/sysconfig/network],v=%d' % version, 'owner': 'root', 'path': '/etc/sysconfig/network', 'permissions': 644, 'purged': False, 'reload': False, 'requires': [], 'version': version }, { 'group': 'root', 'hash': 'b4350bef50c3ec3ee532d4a3f9d6daedec3d2aba', 'id': 'std::File[vm2.dev.inmanta.com,path=/etc/motd],v=%d' % version, 'owner': 'root', 'path': '/etc/motd', 'permissions': 644, 'purged': False, 'reload': False, 'requires': [], 'version': version }, { 'group': 'root', 'hash': '3bfcdad9ab7f9d916a954f1a96b28d31d95593e4', 'id': 'std::File[vm1.dev.inmanta.com,path=/etc/hostname],v=%d' % version, 'owner': 'root', 'path': '/etc/hostname', 'permissions': 644, 'purged': False, 'reload': False, 'requires': [], 'version': version }, { 'id': 'std::Service[vm1.dev.inmanta.com,name=network],v=%d' % version, 'name': 'network', 'onboot': True, 'requires': [ 'std::File[vm1.dev.inmanta.com,path=/etc/sysconfig/network],v=%d' % version ], 'state': 'running', 'version': version }] res = yield client.put_version(tid=environment, version=version, resources=resources, unknowns=[], version_info={}) assert res.code == 200 result = yield client.list_versions(environment) assert result.code == 200 assert result.result["count"] == 1 result = yield client.release_version(environment, version, push=False) assert result.code == 200 result = yield client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["version"] == version assert result.result["model"]["total"] == len(resources) assert result.result["model"]["released"] assert result.result["model"]["result"] == "deploying" result = yield aclient.get_resources_for_agent(environment, "vm1.dev.inmanta.com") assert result.code == 200 assert len(result.result["resources"]) == 3 action_id = uuid.uuid4() now = datetime.now() result = yield aclient.resource_action_update(environment, [ "std::File[vm1.dev.inmanta.com,path=/etc/sysconfig/network],v=%d" % version ], action_id, "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = yield client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["done"] == 1 action_id = uuid.uuid4() now = datetime.now() result = yield aclient.resource_action_update( environment, ["std::File[vm1.dev.inmanta.com,path=/etc/hostname],v=%d" % version], action_id, "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = yield client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["done"] == 2
async def test_purge_on_delete_compile_failed(client, server, clienthelper, environment): """ Test purge on delete of resources """ agent = Agent("localhost", {"blah": "localhost"}, environment=environment, code_loader=False) await agent.start() aclient = agent._client version = await clienthelper.get_version() resources = [ { "group": "root", "hash": "89bf880a0dc5ffc1156c8d958b4960971370ee6a", "id": "std::File[vm1,path=/tmp/file1],v=%d" % version, "owner": "root", "path": "/tmp/file1", "permissions": 644, "purged": False, "reload": False, "requires": [], "purge_on_delete": True, "version": version, }, { "group": "root", "hash": "b4350bef50c3ec3ee532d4a3f9d6daedec3d2aba", "id": "std::File[vm1,path=/tmp/file2],v=%d" % version, "owner": "root", "path": "/tmp/file2", "permissions": 644, "purged": False, "reload": False, "purge_on_delete": True, "requires": ["std::File[vm1,path=/tmp/file1],v=%d" % version], "version": version, }, { "group": "root", "hash": "89bf880a0dc5ffc1156c8d958b4960971370ee6a", "id": "std::File[vm1,path=/tmp/file3],v=%d" % version, "owner": "root", "path": "/tmp/file3", "permissions": 644, "purged": False, "reload": False, "requires": [], "purge_on_delete": True, "version": version, }, ] await clienthelper.put_version_simple(resources, version) # Release the model and set all resources as deployed result = await client.release_version(environment, version, False) assert result.code == 200 now = datetime.now() result = await aclient.resource_action_update( environment, ["std::File[vm1,path=/tmp/file1],v=%d" % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = await aclient.resource_action_update( environment, ["std::File[vm1,path=/tmp/file2],v=%d" % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = await aclient.resource_action_update( environment, ["std::File[vm1,path=/tmp/file3],v=%d" % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = await client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["version"] == version assert result.result["model"]["total"] == len(resources) assert result.result["model"]["done"] == len(resources) assert result.result["model"]["released"] assert result.result["model"]["result"] == const.VersionState.success.name # New version with only file3 version = await clienthelper.get_version() result = await client.put_version( tid=environment, version=version, resources=[], unknowns=[{ "parameter": "a", "source": "b" }], version_info={ const.EXPORT_META_DATA: { const.META_DATA_COMPILE_STATE: const.Compilestate.failed } }, compiler_version=get_compiler_version(), ) assert result.code == 200 result = await client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["total"] == 0 await agent.stop() assert len(result.result["unknowns"]) == 1
async def test_purge_on_delete_ignore(client, clienthelper, server, environment): """ Test purge on delete behavior for resources that have not longer purged_on_delete set """ agent = Agent("localhost", {"blah": "localhost"}, environment=environment, code_loader=False) await agent.start() aclient = agent._client # Version 1 with purge_on_delete true version = await clienthelper.get_version() resources = [{ "group": "root", "hash": "89bf880a0dc5ffc1156c8d958b4960971370ee6a", "id": "std::File[vm1,path=/tmp/file1],v=%d" % version, "owner": "root", "path": "/tmp/file1", "permissions": 644, "purged": False, "reload": False, "requires": [], "purge_on_delete": True, "version": version, }] res = await client.put_version( tid=environment, version=version, resources=resources, unknowns=[], version_info={}, compiler_version=get_compiler_version(), ) assert res.code == 200 # Release the model and set all resources as deployed result = await client.release_version(environment, version, False) assert result.code == 200 now = datetime.now() result = await aclient.resource_action_update( environment, ["std::File[vm1,path=/tmp/file1],v=%d" % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = await client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["version"] == version assert result.result["model"]["total"] == len(resources) assert result.result["model"]["done"] == len(resources) assert result.result["model"]["released"] assert result.result["model"]["result"] == const.VersionState.success.name # Version 2 with purge_on_delete false version = await clienthelper.get_version() resources = [{ "group": "root", "hash": "89bf880a0dc5ffc1156c8d958b4960971370ee6a", "id": "std::File[vm1,path=/tmp/file1],v=%d" % version, "owner": "root", "path": "/tmp/file1", "permissions": 644, "purged": False, "reload": False, "requires": [], "purge_on_delete": False, "version": version, }] res = await client.put_version( tid=environment, version=version, resources=resources, unknowns=[], version_info={}, compiler_version=get_compiler_version(), ) assert res.code == 200 # Release the model and set all resources as deployed result = await client.release_version(environment, version, False) assert result.code == 200 now = datetime.now() result = await aclient.resource_action_update( environment, ["std::File[vm1,path=/tmp/file1],v=%d" % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = await client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["version"] == version assert result.result["model"]["total"] == len(resources) assert result.result["model"]["done"] == len(resources) assert result.result["model"]["released"] assert result.result["model"]["result"] == const.VersionState.success.name # Version 3 with no resources version = await clienthelper.get_version() resources = [] res = await client.put_version( tid=environment, version=version, resources=resources, unknowns=[], version_info={}, compiler_version=get_compiler_version(), ) assert res.code == 200 result = await client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["version"] == version assert result.result["model"]["total"] == len(resources) await agent.stop()
async def test_send_events_cross_agent_deploying(resource_container, environment, server, client, no_agent_backoff, async_finalizer, clienthelper): """ Send and receive events over agents """ agentmanager = server.get_slice(SLICE_AGENT_MANAGER) resource_container.Provider.reset() agent = Agent(hostname="node1", environment=environment, agent_map={"agent1": "localhost"}, code_loader=False) async_finalizer.add(agent.stop) await agent.add_end_point_name("agent1") await agent.start() await retry_limited(lambda: len(agentmanager.sessions) == 1, 10) agent2 = Agent(hostname="node2", environment=environment, agent_map={"agent2": "localhost"}, code_loader=False) async_finalizer.add(agent2.stop) await agent2.add_end_point_name("agent2") await agent2.start() await retry_limited(lambda: len(agentmanager.sessions) == 2, 10) version = await clienthelper.get_version() res_id_1 = "test::Resource[agent1,key=key1],v=%d" % version resources = [ { "key": "key1", "value": "value1", "id": res_id_1, "send_event": False, "purged": False, "requires": ["test::Wait[agent2,key=key2],v=%d" % version], }, { "key": "key2", "value": "value2", "id": "test::Wait[agent2,key=key2],v=%d" % version, "send_event": True, "requires": [], "purged": False, }, ] result = await client.put_version( tid=environment, version=version, resources=resources, unknowns=[], version_info={}, compiler_version=get_compiler_version(), ) assert result.code == 200 # do a deploy result = await client.release_version( environment, version, True, const.AgentTriggerMethod.push_full_deploy) assert result.code == 200 result = await client.get_version(environment, version) assert result.code == 200 await _wait_for_n_deploying(client, environment, version, 1) # restart deploy result = await client.release_version( environment, version, True, const.AgentTriggerMethod.push_full_deploy) assert result.code == 200 await resource_container.wait_for_done_with_waiters( client, environment, version) # incorrect CAD handling causes skip, which completes deploy without writing assert resource_container.Provider.get("agent1", "key1") == "value1"
async def test_send_events_cross_agent_restart(resource_container, environment, server, client, clienthelper, no_agent_backoff, async_finalizer): """ Send and receive events over agents with agents starting after deploy """ agentmanager = server.get_slice(SLICE_AGENT_MANAGER) config.Config.set("config", "agent-deploy-interval", "0") config.Config.set("config", "agent-repair-interval", "0") resource_container.Provider.reset() agent2 = Agent(hostname="node2", environment=environment, agent_map={"agent2": "localhost"}, code_loader=False) async_finalizer.add(agent2.stop) await agent2.add_end_point_name("agent2") await agent2.start() await retry_limited(lambda: len(agentmanager.sessions) == 1, 10) version = await clienthelper.get_version() res_id_1 = "test::Resource[agent1,key=key1],v=%d" % version resources = [ { "key": "key1", "value": "value1", "id": res_id_1, "send_event": False, "purged": False, "requires": ["test::Resource[agent2,key=key2],v=%d" % version], }, { "key": "key2", "value": "value2", "id": "test::Resource[agent2,key=key2],v=%d" % version, "send_event": True, "requires": [], "purged": False, }, ] await clienthelper.put_version_simple(resources, version) # do a deploy result = await client.release_version( environment, version, True, const.AgentTriggerMethod.push_full_deploy) assert result.code == 200 result = await client.get_version(environment, version) assert result.code == 200 # wait for agent 2 to finish while (result.result["model"]["total"] - result.result["model"]["done"]) > 1: result = await client.get_version(environment, version) await asyncio.sleep(0.1) assert resource_container.Provider.get("agent2", "key2") == "value2" # start agent 1 and wait for it to finish agent = Agent(hostname="node1", environment=environment, agent_map={"agent1": "localhost"}, code_loader=False) async_finalizer.add(agent.stop) await agent.add_end_point_name("agent1") await agent.start() await retry_limited(lambda: len(agentmanager.sessions) == 2, 10) # Events are only propagated in a full deploy await agent._instances["agent1"].get_latest_version_for_agent( reason="Repair", incremental_deploy=False, is_repair_run=False) await _wait_until_deployment_finishes(client, environment, version) assert resource_container.Provider.get("agent1", "key1") == "value1" events = resource_container.Provider.getevents("agent1", "key1") assert len(events) == 1 for res_id, res in events[0].items(): assert res_id.agent_name == "agent2" assert res_id.attribute_value == "key2" assert res["status"] == const.ResourceState.deployed assert res["change"] == const.Change.created
async def test_resource_update(postgresql_client, client, clienthelper, server, environment): """ Test updating resources and logging """ agent = Agent("localhost", {"blah": "localhost"}, environment=environment, code_loader=False) await agent.start() aclient = agent._client version = await clienthelper.get_version() resources = [] for j in range(10): resources.append({ "group": "root", "hash": "89bf880a0dc5ffc1156c8d958b4960971370ee6a", "id": "std::File[vm1,path=/tmp/file%d],v=%d" % (j, version), "owner": "root", "path": "/tmp/file%d" % j, "permissions": 644, "purged": False, "reload": False, "requires": [], "version": version, }) res = await client.put_version( tid=environment, version=version, resources=resources, unknowns=[], version_info={}, compiler_version=get_compiler_version(), ) assert res.code == 200 result = await client.release_version(environment, version, False) assert result.code == 200 resource_ids = [x["id"] for x in resources] # Start the deploy action_id = uuid.uuid4() now = datetime.now() result = await aclient.resource_action_update( environment, resource_ids, action_id, "deploy", now, status=const.ResourceState.deploying) assert result.code == 200 # Get the status from a resource result = await client.get_resource(tid=environment, id=resource_ids[0], logs=True) assert result.code == 200 logs = {x["action"]: x for x in result.result["logs"]} assert "deploy" in logs assert "finished" not in logs["deploy"] assert "messages" not in logs["deploy"] assert "changes" not in logs["deploy"] # Send some logs result = await aclient.resource_action_update( environment, resource_ids, action_id, "deploy", status=const.ResourceState.deploying, messages=[ data.LogLine.log(const.LogLevel.INFO, "Test log %(a)s %(b)s", a="a", b="b") ], ) assert result.code == 200 # Get the status from a resource result = await client.get_resource(tid=environment, id=resource_ids[0], logs=True) assert result.code == 200 logs = {x["action"]: x for x in result.result["logs"]} assert "deploy" in logs assert "messages" in logs["deploy"] assert len(logs["deploy"]["messages"]) == 1 assert logs["deploy"]["messages"][0]["msg"] == "Test log a b" assert "finished" not in logs["deploy"] assert "changes" not in logs["deploy"] # Finish the deploy now = datetime.now() changes = { x: { "owner": { "old": "root", "current": "inmanta" } } for x in resource_ids } result = await aclient.resource_action_update(environment, resource_ids, action_id, "deploy", finished=now, changes=changes) assert result.code == 400 result = await aclient.resource_action_update( environment, resource_ids, action_id, "deploy", status=const.ResourceState.deployed, finished=now, changes=changes) assert result.code == 200 result = await client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["done"] == 10 await agent.stop()
async def test_get_resource_for_agent(server_multi, client_multi, environment_multi): """ Test the server to manage the updates on a model during agent deploy """ agent = Agent("localhost", {"nvblah": "localhost"}, environment=environment_multi, code_loader=False) await agent.add_end_point_name("vm1.dev.inmanta.com") await agent.add_end_point_name("vm2.dev.inmanta.com") await agent.start() aclient = agent._client version = (await client_multi.reserve_version(environment_multi)).result["data"] resources = [ { "group": "root", "hash": "89bf880a0dc5ffc1156c8d958b4960971370ee6a", "id": "std::File[vm1.dev.inmanta.com,path=/etc/sysconfig/network],v=%d" % version, "owner": "root", "path": "/etc/sysconfig/network", "permissions": 644, "purged": False, "reload": False, "requires": [], "version": version, }, { "group": "root", "hash": "b4350bef50c3ec3ee532d4a3f9d6daedec3d2aba", "id": "std::File[vm2.dev.inmanta.com,path=/etc/motd],v=%d" % version, "owner": "root", "path": "/etc/motd", "permissions": 644, "purged": False, "reload": False, "requires": [], "version": version, }, { "group": "root", "hash": "3bfcdad9ab7f9d916a954f1a96b28d31d95593e4", "id": "std::File[vm1.dev.inmanta.com,path=/etc/hostname],v=%d" % version, "owner": "root", "path": "/etc/hostname", "permissions": 644, "purged": False, "reload": False, "requires": [], "version": version, }, { "id": "std::Service[vm1.dev.inmanta.com,name=network],v=%d" % version, "name": "network", "onboot": True, "requires": [ "std::File[vm1.dev.inmanta.com,path=/etc/sysconfig/network],v=%d" % version ], "state": "running", "version": version, }, ] res = await client_multi.put_version( tid=environment_multi, version=version, resources=resources, unknowns=[], version_info={}, compiler_version=get_compiler_version(), ) assert res.code == 200 result = await client_multi.list_versions(environment_multi) assert result.code == 200 assert result.result["count"] == 1 result = await client_multi.release_version(environment_multi, version, False) assert result.code == 200 result = await client_multi.get_version(environment_multi, version) assert result.code == 200 assert result.result["model"]["version"] == version assert result.result["model"]["total"] == len(resources) assert result.result["model"]["released"] assert result.result["model"]["result"] == "deploying" result = await aclient.get_resources_for_agent(environment_multi, "vm1.dev.inmanta.com") assert result.code == 200 assert len(result.result["resources"]) == 3 action_id = uuid.uuid4() now = datetime.now() result = await aclient.resource_action_update( environment_multi, [ "std::File[vm1.dev.inmanta.com,path=/etc/sysconfig/network],v=%d" % version ], action_id, "deploy", now, now, "deployed", [], {}, ) assert result.code == 200 result = await client_multi.get_version(environment_multi, version) assert result.code == 200 assert result.result["model"]["done"] == 1 action_id = uuid.uuid4() now = datetime.now() result = await aclient.resource_action_update( environment_multi, ["std::File[vm1.dev.inmanta.com,path=/etc/hostname],v=%d" % version], action_id, "deploy", now, now, "deployed", [], {}, ) assert result.code == 200 result = await client_multi.get_version(environment_multi, version) assert result.code == 200 assert result.result["model"]["done"] == 2 await agent.stop()
async def test_disable_purge_on_delete(client, clienthelper, server, environment): """ Test disable purge on delete of resources """ agent = Agent("localhost", {"blah": "localhost"}, environment=environment, code_loader=False) await agent.start() aclient = agent._client env = await data.Environment.get_by_id(environment) await env.set(data.PURGE_ON_DELETE, False) version = await clienthelper.get_version() resources = [{ "group": "root", "hash": "89bf880a0dc5ffc1156c8d958b4960971370ee6a", "id": "std::File[vm1,path=/tmp/file1],v=%d" % version, "owner": "root", "path": "/tmp/file1", "permissions": 644, "purged": False, "reload": False, "requires": [], "purge_on_delete": True, "version": version, }] res = await client.put_version( tid=environment, version=version, resources=resources, unknowns=[], version_info={}, compiler_version=get_compiler_version(), ) assert res.code == 200 # Release the model and set all resources as deployed result = await client.release_version(environment, version, False) assert result.code == 200 now = datetime.now() result = await aclient.resource_action_update( environment, ["std::File[vm1,path=/tmp/file1],v=%d" % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = await client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["result"] == const.VersionState.success.name # Empty version version = await clienthelper.get_version() result = await client.put_version(tid=environment, version=version, resources=[], unknowns=[], version_info={}, compiler_version=get_compiler_version()) assert result.code == 200 result = await client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["total"] == 0 await agent.stop()
async def test_purge_on_delete(client, clienthelper, server, environment): """ Test purge on delete of resources """ agent = Agent("localhost", {"blah": "localhost"}, environment=environment, code_loader=False) await agent.start() aclient = agent._client version = await clienthelper.get_version() resources = [ { "group": "root", "hash": "89bf880a0dc5ffc1156c8d958b4960971370ee6a", "id": "std::File[vm1,path=/tmp/file1],v=%d" % version, "owner": "root", "path": "/tmp/file1", "permissions": 644, "purged": False, "reload": False, "requires": [], "purge_on_delete": True, "version": version, }, { "group": "root", "hash": "b4350bef50c3ec3ee532d4a3f9d6daedec3d2aba", "id": "std::File[vm1,path=/tmp/file2],v=%d" % version, "owner": "root", "path": "/tmp/file2", "permissions": 644, "purged": False, "reload": False, "purge_on_delete": True, "requires": ["std::File[vm1,path=/tmp/file1],v=%d" % version], "version": version, }, { "group": "root", "hash": "89bf880a0dc5ffc1156c8d958b4960971370ee6a", "id": "std::File[vm1,path=/tmp/file3],v=%d" % version, "owner": "root", "path": "/tmp/file3", "permissions": 644, "purged": False, "reload": False, "requires": [], "purge_on_delete": True, "version": version, }, ] res = await client.put_version( tid=environment, version=version, resources=resources, unknowns=[], version_info={}, compiler_version=get_compiler_version(), ) assert res.code == 200 # Release the model and set all resources as deployed result = await client.release_version(environment, version, False) assert result.code == 200 now = datetime.now() result = await aclient.resource_action_update( environment, ["std::File[vm1,path=/tmp/file1],v=%d" % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = await aclient.resource_action_update( environment, ["std::File[vm1,path=/tmp/file2],v=%d" % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = await aclient.resource_action_update( environment, ["std::File[vm1,path=/tmp/file3],v=%d" % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = await client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["version"] == version assert result.result["model"]["total"] == len(resources) assert result.result["model"]["done"] == len(resources) assert result.result["model"]["released"] assert result.result["model"]["result"] == const.VersionState.success.name # New version with only file3 version = await clienthelper.get_version() res3 = { "group": "root", "hash": "89bf880a0dc5ffc1156c8d958b4960971370ee6a", "id": "std::File[vm1,path=/tmp/file3],v=%d" % version, "owner": "root", "path": "/tmp/file3", "permissions": 644, "purged": False, "reload": False, "requires": [], "purge_on_delete": True, "version": version, } result = await client.put_version( tid=environment, version=version, resources=[res3], unknowns=[], version_info={}, compiler_version=get_compiler_version(), ) assert result.code == 200 result = await client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["total"] == 3 # validate requires and provides file1 = [x for x in result.result["resources"] if "file1" in x["id"]][0] file2 = [x for x in result.result["resources"] if "file2" in x["id"]][0] file3 = [x for x in result.result["resources"] if "file3" in x["id"]][0] assert file1["attributes"]["purged"] assert file2["attributes"]["purged"] assert not file3["attributes"]["purged"] await agent.stop()
async def test_send_events_cross_agent(resource_container, environment, server, client, async_finalizer, clienthelper): """ Send and receive events over agents """ agentmanager = server.get_slice(SLICE_AGENT_MANAGER) resource_container.Provider.reset() agent = Agent(hostname="node1", environment=environment, agent_map={"agent1": "localhost"}, code_loader=False) async_finalizer.add(agent.stop) await agent.add_end_point_name("agent1") await agent.start() await retry_limited(lambda: len(agentmanager.sessions) == 1, 10) agent2 = Agent(hostname="node2", environment=environment, agent_map={"agent2": "localhost"}, code_loader=False) async_finalizer.add(agent2.stop) await agent2.add_end_point_name("agent2") await agent2.start() await retry_limited(lambda: len(agentmanager.sessions) == 2, 10) version = await clienthelper.get_version() res_id_1 = "test::Resource[agent1,key=key1],v=%d" % version resources = [ { "key": "key1", "value": "value1", "id": res_id_1, "send_event": False, "purged": False, "requires": ["test::Resource[agent2,key=key2],v=%d" % version], }, { "key": "key2", "value": "value2", "id": "test::Resource[agent2,key=key2],v=%d" % version, "send_event": True, "requires": [], "purged": False, }, ] result = await client.put_version( tid=environment, version=version, resources=resources, unknowns=[], version_info={}, compiler_version=get_compiler_version(), ) assert result.code == 200 # do a deploy result = await client.release_version( environment, version, True, const.AgentTriggerMethod.push_full_deploy) assert result.code == 200 result = await client.get_version(environment, version) assert result.code == 200 await _wait_until_deployment_finishes(client, environment, version) assert resource_container.Provider.get("agent1", "key1") == "value1" assert resource_container.Provider.get("agent2", "key2") == "value2" events = resource_container.Provider.getevents("agent1", "key1") assert len(events) == 1 for res_id, res in events[0].items(): assert res_id.agent_name == "agent2" assert res_id.attribute_value == "key2" assert res["status"] == const.ResourceState.deployed assert res["change"] == const.Change.created
async def test_send_events_cross_agent_unavailable(resource_container, environment, server, client, clienthelper, no_agent_backoff, async_finalizer, caplog): """ Having unavailable cross agent dependencies shouldn't result in invalid Event objects (#2501) """ agentmanager = server.get_slice(SLICE_AGENT_MANAGER) config.Config.set("config", "agent-deploy-interval", "0") config.Config.set("config", "agent-repair-interval", "0") resource_container.Provider.reset() agent2 = Agent(hostname="node2", environment=environment, agent_map={"agent2": "localhost"}, code_loader=False) async_finalizer.add(agent2.stop) await agent2.add_end_point_name("agent2") await agent2.start() await retry_limited(lambda: len(agentmanager.sessions) == 1, 10) agent = Agent(hostname="node1", environment=environment, agent_map={"agent1": "localhost"}, code_loader=False) async_finalizer.add(agent.stop) await agent.add_end_point_name("agent1") await agent.start() await retry_limited(lambda: len(agentmanager.sessions) == 2, 10) version = await clienthelper.get_version() res_id_1 = "test::Resource[agent1,key=key1],v=%d" % version res_id_2 = "test::ResourceNoHandler[agent2,key=key2],v=%d" % version res_id_3 = "test::Resource[agent2,key=key3],v=%d" % version # resource 1 requires an available and an unavailable resource resources = [ { "key": "key1", "value": "value1", "id": res_id_1, "send_event": True, "requires": [res_id_2, res_id_3], "purged": False, }, { "key": "key2", "value": "value2", "id": res_id_2, "send_event": True, "requires": [], "purged": False, }, { "key": "key3", "value": "value3", "id": res_id_3, "send_event": True, "requires": [], "purged": False, }, ] await clienthelper.put_version_simple(resources, version) # do a deploy result = await client.release_version( environment, version, True, const.AgentTriggerMethod.push_full_deploy) assert result.code == 200 result = await client.get_version(environment, version) assert result.code == 200 await _wait_until_deployment_finishes(client, environment, version) async def check_resource_state(environment, id, state): result = await client.get_resource(tid=environment, id=id, logs=False, status=True) assert result.code == 200 return result.result["status"] == state assert await check_resource_state(environment, res_id_1, "skipped") assert await check_resource_state(environment, res_id_2, "unavailable") assert await check_resource_state(environment, res_id_3, "deployed") log_doesnt_contain( caplog, "inmanta.util", logging.ERROR, "An exception occurred while handling a future: 1 validation error for Event" )
async def test_purge_on_delete_requires(client, server, environment, clienthelper): """ Test purge on delete of resources and inversion of requires """ agent = Agent("localhost", {"blah": "localhost"}, environment=environment, code_loader=False) await agent.start() aclient = agent._client version = await clienthelper.get_version() resources = [ { "group": "root", "hash": "89bf880a0dc5ffc1156c8d958b4960971370ee6a", "id": "std::File[vm1,path=/tmp/file1],v=%d" % version, "owner": "root", "path": "/tmp/file1", "permissions": 644, "purged": False, "reload": False, "requires": [], "purge_on_delete": True, "version": version, }, { "group": "root", "hash": "b4350bef50c3ec3ee532d4a3f9d6daedec3d2aba", "id": "std::File[vm2,path=/tmp/file2],v=%d" % version, "owner": "root", "path": "/tmp/file2", "permissions": 644, "purged": False, "reload": False, "purge_on_delete": True, "requires": ["std::File[vm1,path=/tmp/file1],v=%d" % version], "version": version, }, ] await clienthelper.put_version_simple(resources, version) # Release the model and set all resources as deployed result = await client.release_version(environment, version, False) assert result.code == 200 now = datetime.now() result = await aclient.resource_action_update( environment, ["std::File[vm1,path=/tmp/file1],v=%d" % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = await aclient.resource_action_update( environment, ["std::File[vm2,path=/tmp/file2],v=%d" % version], uuid.uuid4(), "deploy", now, now, "deployed", [], {}) assert result.code == 200 result = await client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["version"] == version assert result.result["model"]["total"] == len(resources) assert result.result["model"]["done"] == len(resources) assert result.result["model"]["released"] assert result.result["model"]["result"] == const.VersionState.success.name # validate requires and provides file1 = [x for x in result.result["resources"] if "file1" in x["id"]][0] file2 = [x for x in result.result["resources"] if "file2" in x["id"]][0] assert file2["id"] in file1["provides"] assert len(file1["attributes"]["requires"]) == 0 assert len(file2["provides"]) == 0 assert file1["id"] in file2["attributes"]["requires"] result = await client.decomission_environment(id=environment, metadata={ "message": "test", "type": "test" }) assert result.code == 200 version = result.result["version"] result = await client.get_version(environment, version) assert result.code == 200 assert result.result["model"]["total"] == len(resources) # validate requires and provides file1 = [x for x in result.result["resources"] if "file1" in x["id"]][0] file2 = [x for x in result.result["resources"] if "file2" in x["id"]][0] assert file2["id"] in file1["attributes"]["requires"] assert type(file1["attributes"]["requires"]) == list assert len(file1["provides"]) == 0 assert len(file2["attributes"]["requires"]) == 0 assert file1["id"] in file2["provides"] await agent.stop()