def test_deploy(self): # make sure this works f = urllib.request.urlopen( "http://localhost:8010/fixtures/helmrepo/index.yaml") f.close() runner = Runner(YamlManifest(manifest)) run1 = runner.run(JobOptions(dryrun=False, verbose=3, startTime=1)) assert not run1.unexpectedAbort, run1.unexpectedAbort.getStackTrace() summary = run1.jsonSummary() # runner.manifest.statusSummary() # print(summary) self.assertEqual( summary["job"], { "id": "A01110000000", "status": "ok", "total": 4, "ok": 4, "error": 0, "unknown": 0, "skipped": 0, "changed": 4, }, ) assert all(task["targetStatus"] == "ok" for task in summary["tasks"]), summary["tasks"]
def test_discover(self): path = __file__ + "/../examples/helm-manifest.yaml" manifest = YamlManifest(path=path) runner = Runner(manifest) assert not manifest.lastJob, "expected new manifest" output = six.StringIO() # so we don't save the file job = runner.run(JobOptions(workflow="discover", out=output, startTime=1)) # print(job.summary()) # print("discovered", runner.manifest.tosca.discovered) # print("discovered manifest", output.getvalue()) assert not job.unexpectedAbort, job.unexpectedAbort.getStackTrace() baseDir = __file__ + "/../examples/" manifest2 = YamlManifest(output.getvalue(), path=baseDir) manifest2.manifest.path = os.path.abspath( path ) # set the basedir which sets the current working dir # manifest2.statusSummary() output2 = six.StringIO() job2 = Runner(manifest2).run( JobOptions(workflow="discover", out=output2, startTime=2) ) # print("2", output2.getvalue()) # print('job2', job2.summary()) assert not job2.unexpectedAbort, job2.unexpectedAbort.getStackTrace() # print("job", json.dumps(job2.jsonSummary(), indent=2)) # should not have found any tasks to run: assert len(job2.workDone) == 8, list(job2.workDone)
def test_result_template_errors(caplog): manifest = """\ apiVersion: unfurl/v1alpha1 kind: Ensemble spec: service_template: topology_template: node_templates: testNode: type: tosca.nodes.Root interfaces: Standard: operations: configure: implementation: className: unfurl.configurators.TemplateConfigurator inputs: resultTemplate: | - name: .self attributes: outputVar: "{{ SELF.missing }}" """ runner = Runner(YamlManifest(manifest)) job = runner.run() assert not job.unexpectedAbort, job.unexpectedAbort.get_stack_trace() for record in caplog.records: if record.levelname == "WARNING": assert ( record.getMessage() == 'error processing resultTemplate for testNode: <<Error rendering template: missing attribute or key: "missing">>' ) break else: assert False, "log message not found"
def test_fileRef(self): simple = (""" apiVersion: %s kind: Manifest spec: service_template: topology_template: node_templates: test: type: tosca.nodes.Root properties: file: eval: file: foo.txt interfaces: Standard: create: implementation: FileTest inputs: path: eval: file::path contents: eval: file::contents """ % API_VERSION) cliRunner = CliRunner() with cliRunner.isolated_filesystem(): # as tmpDir manifest = YamlManifest(simple, path=".") runner = Runner(manifest) output = six.StringIO() with open("foo.txt", "w") as f: f.write("test") job = runner.run(JobOptions(add=True, out=output, startTime="test")) task = list(job.workDone.values())[0] self.assertEqual(task.result.result, "foo.txt")
def _runInputAndOutputs(self, manifest): job = Runner(manifest).run(JobOptions(add=True, startTime="time-to-test")) assert not job.unexpectedAbort, job.unexpectedAbort.get_stack_trace() my_server = manifest.get_root_resource().find_resource("my_server") assert my_server self.assertEqual( "10 GB", my_server.query({"get_property": ["SELF", "host", "disk_size"]}) ) assert my_server.attributes["test"] == "cpus: 2" assert my_server.attributes["concat2"] is None # print(job.out.getvalue()) testSensitive = manifest.get_root_resource().find_resource("testSensitive") for name, toscaType in ( ("access_token", "tosca.datatypes.Credential"), ("TEST_VAR", "unfurl.datatypes.EnvVar"), ): assert testSensitive.template.propertyDefs[name].schema["type"] == toscaType def t(datatype): return datatype.type == "unfurl.datatypes.EnvVar" envvars = set(testSensitive.template.find_props(testSensitive.attributes, t)) self.assertEqual(envvars, set([("TEST_VAR", "foo"), ("VAR1", "more")])) outputIp = job.get_outputs()["server_ip"] self.assertEqual(outputIp, "10.0.0.1") assert isinstance(outputIp, sensitive_str), type(outputIp) assert job.status == Status.ok, job.summary() self.assertEqual("RHEL", testSensitive.attributes["distribution"]) return outputIp, job
def test_login(self): """ test that runner figures out the proper tasks to run """ import docker client = docker.from_env() assert client, "docker not installed?" runner = Runner(YamlManifest(manifest)) # pickled = pickle.dumps(runner.manifest, -1) # manifest2 = pickle.loads(pickled) run1 = runner.run(JobOptions(instance="test1")) assert len(run1.workDone) == 1, run1.workDone tasks = list(run1.workDone.values()) # docker login will fail because user doesn't exist: assert tasks[0].status.name == "error", tasks[0].status self.assertIn("401 Client Error", tasks[0].result.result.get("msg", "")) # but the repository and image path will have been created self.assertEqual( tasks[0].result.outputs.get("image_path"), "index.docker.io/repo/image", ) registry = tasks[0].result.outputs.get("registry") assert registry and isinstance(registry, toscaparser.repositories.Repository) assert not run1.unexpectedAbort, run1.unexpectedAbort.get_stack_trace()
def lifecycle( manifest: Manifest, steps: Optional[Iterable[Step]] = DEFAULT_STEPS) -> Iterable[Job]: runner = Runner(manifest) for i, step in enumerate(steps, start=1): print(f"starting step #{i} - {step.workflow}") job = runner.run(JobOptions(workflow=step.workflow, starttime=i)) yield _check_job(job, i, step)
def test_timeout_with_ensemble(self): runner = Runner(YamlManifest(ENSEMBLE_TIMEOUT)) start_time = datetime.now() job = runner.run(JobOptions(instance="test_node")) delta = datetime.now() - start_time assert job.status == Status.error assert delta < timedelta(seconds=2), delta - timedelta(seconds=2)
def test_run_without_dry_run(self, command, dryrun): ensemble = ENSEMBLE_DRY_RUN.format(command=command, dryrun=dryrun) runner = Runner(YamlManifest(ensemble)) job = runner.run(JobOptions(instance="test_node", dryrun=False)) assert job.status == Status.ok task = list(job.workDone.values())[0] cmd = task.result.result["cmd"].strip() assert cmd == "echo hello world"
def test_error_if_dry_run_not_defined_for_task(self): ensemble = ENSEMBLE_DRY_RUN.format(command="command: echo hello world", dryrun="") runner = Runner(YamlManifest(ensemble)) job = runner.run(JobOptions(instance="test_node", dryrun=True)) task = list(job.workDone.values())[0] assert job.status == Status.error assert task.result.result == "could not run: dry run not supported"
def test_exclusive(self, manager_sync): runner = Runner(YamlManifest(ENSEMBLE_EXCLUSIVE)) job = runner.run(JobOptions(workflow="deploy")) assert job.status == Status.ok node = job.rootResource.find_resource("test_node") # records are replaced by instance assert len(node.attributes["zone"]) == 1 assert manager_sync.called
def test_check(self): runner = Runner(YamlManifest(ENSEMBLE_ROUTE53)) runner.run(JobOptions(workflow="deploy")) job = runner.run(JobOptions(workflow="check")) assert job.status == Status.ok task = list(job.workDone.values())[0] # this means that dns records were correctly set during deploy: assert task.target_status == Status.ok assert task.result.result == "DNS records in sync"
def test_shell(self): """ test that runner figures out the proper tasks to run """ runner = Runner(YamlManifest(self.ensemble)) job = runner.run(JobOptions(instance="test1")) assert len(job.workDone) == 1, job.workDone node = runner.manifest.get_root_resource().find_resource("test1") assert node.attributes["stdout"] == "helloworld" assert not job.unexpectedAbort, job.unexpectedAbort.get_stack_trace()
def test_configurator(self): """ test that runner figures out the proper tasks to run """ runner = Runner(YamlManifest(manifest)) run1 = runner.run(JobOptions(resource="test1")) assert not run1.unexpectedAbort, run1.unexpectedAbort.getStackTrace() assert len(run1.workDone) == 1, run1.workDone result = list(run1.workDone.values())[0].result self.assertEqual(result.outputs, {"fact1": "test1", "fact2": "test"}) self.assertEqual(result.result.get("stdout"), sys.executable) assert run1.status == Status.ok, run1.summary()
def test_delete(self): runner = Runner(YamlManifest(ENSEMBLE_ROUTE53)) job = runner.run(JobOptions(workflow="deploy")) assert job.status == Status.ok assert not job.unexpectedAbort, job.unexpectedAbort.get_stack_trace() node = job.rootResource.find_resource("test_node") assert node and len(node.attributes["zone"]) == 2 job = runner.run(JobOptions(workflow="undeploy")) assert job.status == Status.ok assert not job.unexpectedAbort, job.unexpectedAbort.get_stack_trace() node = job.rootResource.find_resource("test_node") assert dict(node.attributes["zone"]) == {}
def test_neededTasks(self): """ test that runner figures out the proper tasks to run """ runner = Runner(YamlManifest(manifest)) test1 = runner.manifest.get_root_resource().find_resource("test1") assert test1 # missing = runner.manifest.spec[0].findInvalidPreconditions(test1) # assert not missing, missing run1 = runner.run(JobOptions(instance="test1")) assert not run1.unexpectedAbort, run1.unexpectedAbort.get_stack_trace() assert len(run1.workDone) == 1, run1.summary()
def test_k8sConfig(self): os.environ["TEST_SECRET"] = "a secret" manifest = YamlManifest(manifestScript) job = Runner(manifest).run(JobOptions(add=True, startTime=1)) assert not job.unexpectedAbort assert job.status == Status.ok, job.summary() # print(job.summary()) # print(job.out.getvalue()) # verify secret contents isn't saved in config self.assertNotIn("a secret", job.out.getvalue()) self.assertNotIn("YSBzZWNyZXQ", job.out.getvalue()) # base64 of "a secret" # print (job.out.getvalue()) self.assertIn("<<REDACTED>>", job.out.getvalue()) assert not job.unexpectedAbort assert job.status == Status.ok, job.summary() manifest = YamlManifest(job.out.getvalue()) job2 = Runner(manifest).run( JobOptions(workflow="undeploy", startTime=2)) results = job2.jsonSummary() assert not job2.unexpectedAbort assert job2.status == Status.ok, job2.summary() assert len(results["tasks"]) == 1, results
def test_preConditions(self): """test that the configuration only runs if the resource meets the requirements""" runner = Runner(YamlManifest(manifest)) test1 = runner.manifest.get_root_resource().find_resource("test1") assert test1 self.assertEqual(test1.attributes["meetsTheRequirement"], "copy") # notYetProvided = runner.manifest.spec[0].findMissingProvided(test1) # self.assertEqual(str(notYetProvided[1]), # """[<ValidationError: "'copyOfMeetsTheRequirement' is a required property">]""") jobOptions1 = JobOptions(instance="test1", startTime=1) run1 = runner.run(jobOptions1) # print(run1.out.getvalue()) assert not run1.unexpectedAbort, run1.unexpectedAbort.get_stack_trace() self.assertEqual(test1.attributes["copyOfMeetsTheRequirement"], "copy") # provided = runner.manifest.spec[0].findMissingProvided(test1) # assert not provided, provided # check that the modifications were recorded self.assertEqual( runner.manifest.manifest.config["changes"][0]["changes"], {"::test1": { "copyOfMeetsTheRequirement": "copy" }}, ) test2 = runner.manifest.get_root_resource().find_resource("test2") assert test2 requiredAttribute = test2.attributes["meetsTheRequirement"] assert requiredAttribute is False, requiredAttribute # missing = runner.manifest.specs[0].findMissingRequirements(test2) # self.assertEqual(str(missing[1]), '''[<ValidationError: "False is not of type 'string'">]''') self.verifyRoundtrip(run1.out.getvalue(), jobOptions1) jobOptions2 = JobOptions(instance="test2", startTime=2) run2 = runner.run(jobOptions2) assert not run2.unexpectedAbort, run2.unexpectedAbort.get_stack_trace() assert run2.status == Status.error, run2.status # XXX better error reporting # self.assertEqual(str(run2.problems), "can't run required configuration: resource test2 doesn't meet requirement") # print(run2.out.getvalue()) # don't re-run the failed configurations so nothing will have changed jobOptions2.repair = "none" jobOptions2.skip_new = True self.verifyRoundtrip(run2.out.getvalue(), jobOptions2)
def test_shell(self): """ test that runner figures out the proper tasks to run """ runner = Runner(YamlManifest(manifest)) run1 = runner.run(JobOptions(instance="test1")) assert len(run1.workDone) == 1, run1.workDone self.assertEqual( runner.manifest.getRootResource() .findResource("test1") .attributes["stdout"], "helloworld", ) assert not run1.unexpectedAbort, run1.unexpectedAbort.getStackTrace()
def test_configure(self): runner = Runner(YamlManifest(ENSEMBLE_ROUTE53)) job = runner.run(JobOptions(workflow="deploy")) assert job.status == Status.ok assert not job.unexpectedAbort, job.unexpectedAbort.get_stack_trace() node = job.rootResource.find_resource("test_node") assert node.attributes["zone"][""]["type"] == "A" assert node.attributes["zone"][""]["values"] == [ "2.3.4.5", "2.3.4.6", ] assert node.attributes["zone"]["www"]["values"] == [ "2.3.4.5", "2.3.4.6", ]
def test_ansible(self): """ Run ansible command on a (mock) remote instance. """ try: oldTmpDir = os.environ["UNFURL_TMPDIR"] runner = CliRunner() with runner.isolated_filesystem() as tempDir: os.environ["UNFURL_TMPDIR"] = tempDir path = __file__ + "/../examples/ansible-manifest.yaml" manifest = YamlManifest(path=path) runner = Runner(manifest) output = six.StringIO() # so we don't save the file job = runner.run( JobOptions( workflow="run", host="www.example.com", # this instance doesn't exist so warning is output instance="www.example.com", cmdline=["echo", "foo"], out=output, ) ) assert not job.unexpectedAbort, job.unexpectedAbort.getStackTrace() # verify we wrote out the correct ansible inventory file try: from pathlib import Path p = Path(os.environ["UNFURL_TMPDIR"]) files = list(p.glob("**/*-inventory.yaml")) self.assertEqual(len(files), 1, files) inventory = files[-1] expectedInventory = """all: hosts: www.example.com: ansible_port: 22 ansible_connection: local ansible_user: ubuntu ansible_pipelining: yes ansible_private_key_file: ~/.ssh/example-key.pem vars: {} children: example_group: hosts: {} vars: var_for_ansible_playbook: test children: {} """ with inventory.open() as f: self.assertEqual(f.read(), expectedInventory) except ImportError: pass # skip on 2.7 finally: os.environ["UNFURL_TMPDIR"] = oldTmpDir tasks = list(job.workDone.values()) self.assertEqual(len(tasks), 1) assert "stdout" in tasks[0].result.result, tasks[0].result.result self.assertEqual(tasks[0].result.result["stdout"], "foo")
def test_workflows(self): manifest = YamlManifest( path=__file__ + "/../examples/test-workflow-manifest.yaml" ) # print(manifest.tosca.template.nested_tosca_tpls) self.assertEqual(len(manifest.tosca._workflows), 3) runner = Runner(manifest) output = six.StringIO() job = runner.run( JobOptions(add=True, planOnly=True, out=output, startTime="test") ) # print(job.jsonSummary()) assert not job.unexpectedAbort, job.unexpectedAbort.getStackTrace() self.assertEqual(job.status.name, "ok") self.assertEqual(job.stats()["ok"], 4) self.assertEqual(job.stats()["changed"], 4)
def test_manifest(self): path = __file__ + "/../examples/helm-manifest.yaml" manifest = YamlManifest(path=path) runner = Runner(manifest) assert not manifest.lastJob, "expected new manifest" output = six.StringIO() # so we don't save the file job = runner.run(JobOptions(add=True, out=output, startTime="test")) assert not job.unexpectedAbort, job.unexpectedAbort.getStackTrace() # manifest shouldn't have changed # print("1", output.getvalue()) manifest2 = YamlManifest(output.getvalue()) # manifest2.statusSummary() output2 = six.StringIO() job2 = Runner(manifest2).run( JobOptions(add=True, out=output2, startTime=1)) # print("2", output2.getvalue()) # print(job2.summary()) assert not job2.unexpectedAbort, job2.unexpectedAbort.getStackTrace() # should not have found any tasks to run: assert len(job2.workDone) == 0, job2.workDone self.maxDiff = None # self.assertEqual(output.getvalue(), output2.getvalue()) output3 = six.StringIO() manifest3 = YamlManifest(output2.getvalue()) job3 = Runner(manifest3).run( JobOptions(workflow="undeploy", out=output3, startTime=2)) # print(output3.getvalue()) # only the chart delete task should have ran as it owns the resources it created # print(job3.jsonSummary()) assert len(job3.workDone) == 1, job3.jsonSummary() tasks = list(job3.workDone.values()) assert tasks[0].target.status.name == "absent", tasks[0].target.status
def verifyRoundtrip(self, original, jobOptions): if jobOptions.startTime: jobOptions.startTime += 1 job = Runner(YamlManifest(original)).run(jobOptions) # should not need to run any tasks assert len(job.workDone) == 0, job.workDone self.maxDiff = None self.assertEqual(original, job.out.getvalue())
def test_manifest(self): simple = ( """ apiVersion: %s kind: Manifest spec: instances: anInstance: # template: foo interfaces: Standard: operations: configure: implementation: TestSubtask inputs: {} """ % API_VERSION ) manifest = YamlManifest(simple) runner = Runner(manifest) self.assertEqual(runner.taskCount, 0) output = six.StringIO() job = runner.run(JobOptions(add=True, out=output, startTime="test")) assert not job.unexpectedAbort, job.unexpectedAbort.getStackTrace() # workDone includes subtasks assert len(job.workDone) == 2, job.workDone # manifest shouldn't have changed manifest2 = YamlManifest(output.getvalue()) lock = manifest2.manifest.config["lock"] assert "runtime" in lock and len(lock["repositories"]) == 3 self.assertEqual( manifest2.lastJob["summary"], "2 tasks (2 changed, 2 ok, 0 failed, 0 unknown, 0 skipped)", ) output2 = six.StringIO() job2 = Runner(manifest2).run(JobOptions(add=True, out=output2)) assert not job2.unexpectedAbort, job2.unexpectedAbort.getStackTrace() # should not find any tasks to run assert len(job2.workDone) == 0, job2.workDone self.maxDiff = None self.assertEqual(output.getvalue(), output2.getvalue())
def test_addingResources(self): runner = Runner(YamlManifest(manifest)) jobOptions = JobOptions(instance="test3", startTime=1) run = runner.run(jobOptions) assert not run.unexpectedAbort, run.unexpectedAbort.get_stack_trace() # self.assertEqual(list(run.workDone.keys()), [('test3', 'test'), ('added1', 'config1')]) # print('config', run.out.getvalue()) changes = runner.manifest.manifest.config["changes"][0]["changes"] added = {".added": {"name": "added1", "template": "test1"}} self.assertEqual(changes["::added1"], added) # verify modified self.assertEqual(changes["::test3"], {"copyOfMeetsTheRequirement": "copy"}) # print('test3', run.out.getvalue()) jobOptions.repair = "none" self.verifyRoundtrip(run.out.getvalue(), jobOptions) jobOptions = JobOptions(instance="test4", startTime=2) run = runner.run(jobOptions) assert not run.unexpectedAbort, run.unexpectedAbort.get_stack_trace() self.assertEqual( [(t.name, t.target.name) for t in run.workDone.values()], [ ("for add: Standard.configure", "test4"), ("for add: Standard.configure", "added2"), ], ) # print('test4', run.out.getvalue()) # XXX # verify dependencies added # dependencies = lookupPath( # runner.manifest.manifest.config, # "root.instances.test4.status.configurations.test.dependencies".split("."), # ) # self.assertEqual(dependencies, [{"ref": "::added2"}]) jobOptions.repair = "none" self.verifyRoundtrip(run.out.getvalue(), jobOptions)
def test_container(self): """ test that runner figures out the proper tasks to run """ import docker client = docker.from_env() assert client, "docker not installed?" runner = Runner(YamlManifest(manifest)) # pickled = pickle.dumps(runner.manifest, -1) # manifest2 = pickle.loads(pickled) run1 = runner.run(JobOptions(check=True, template="container1")) # configure (start op shouldn't run since docker_container sets state to started) assert len(run1.workDone) == 2, run1.workDone tasks = list(run1.workDone.values()) assert not tasks[1].target.attributes.get( "container"), "testing that container property isn't required" # print([task.result.outputs for task in tasks]) container = tasks[1].result.outputs.get("container") assert container self.assertEqual(container["Name"], "/test_docker") self.assertEqual(container["State"]["Status"], "exited") self.assertEqual(container["Config"]["Image"], "busybox") self.assertIn("hello", container["Output"].strip()) assert tasks[0].status.name == "ok", tasks[0].status assert tasks[1].status.name == "ok", tasks[1].status assert not run1.unexpectedAbort, run1.unexpectedAbort.get_stack_trace() assert tasks[0].target.status.name == "ok", tasks[0].target.status assert tasks[1].target.status.name == "ok", tasks[1].target.status run2 = runner.run( JobOptions(workflow="undeploy", template="container1")) # stop op shouldn't be called, just delete assert len(run2.workDone) == 1, run2.workDone assert not run2.unexpectedAbort, run2.unexpectedAbort.get_stack_trace() tasks = list(run2.workDone.values()) # runner.manifest.dump() assert tasks[0].status.name == "ok", tasks[0].status assert tasks[0].target.status.name == "absent", tasks[0].target.status
def test_TemplateConfigurator(self): manifest = """\ apiVersion: unfurl/v1alpha1 kind: Manifest spec: service_template: topology_template: node_templates: testNode: type: tosca.nodes.Root interfaces: Standard: operations: configure: implementation: className: unfurl.configurators.TemplateConfigurator inputs: done: result: outputVar: true resultTemplate: | - name: .self attributes: outputVar: "{{ outputVar }}" """ runner = Runner(YamlManifest(manifest)) job = runner.run() assert not job.unexpectedAbort, job.unexpectedAbort.get_stack_trace() self.assertEqual( job.stats(), { "total": 1, "ok": 1, "error": 0, "unknown": 0, "skipped": 0, "changed": 1 }, ) assert job.rootResource.find_resource( "testNode").attributes["outputVar"]
def test_deploy(self): # make sure this works f = urllib.request.urlopen( "http://localhost:8010/fixtures/helmrepo/index.yaml") f.close() runner = Runner(YamlManifest(manifest)) run1 = runner.run(JobOptions(planOnly=True, verbose=3, startTime=1)) mysql_release = runner.manifest.rootResource.findResource( "mysql_release") query = ".::.requirements::[.name=host]::.target::name" res = mysql_release.query(query) assert res == 'unfurl-helm-unittest' runner = Runner(YamlManifest(manifest)) run1 = runner.run(JobOptions(dryrun=False, verbose=3, startTime=1)) assert not run1.unexpectedAbort, run1.unexpectedAbort.getStackTrace() summary = run1.jsonSummary() # runner.manifest.statusSummary() # print(summary) self.assertEqual( summary["job"], { "id": "A01110000000", "status": "ok", "total": 4, "ok": 4, "error": 0, "unknown": 0, "skipped": 0, "changed": 4, }, ) assert all(task["targetStatus"] == "ok" for task in summary["tasks"]), summary["tasks"]
def test_localhost(): runner = Runner(YamlManifest(ensemble)) run1 = runner.run() assert not run1.unexpectedAbort, run1.unexpectedAbort.get_stack_trace() assert len(run1.workDone) == 1, run1.summary() assert run1.json_summary()["job"]["ok"] == 1, run1.summary() localhost = runner.manifest.get_root_resource().find_resource("localhost") assert localhost if localhost.attributes["os_type"] == "Darwin": assert ( localhost.attributes["package_manager"] == "homebrew" ), localhost.attributes else: # assume running on a linus localhost.attributes["package_manager"] in [ "apt", "yum", "rpm", ], localhost.attributes["package_manager"] assert localhost.attributes["architecture"] assert localhost.attributes["architecture"] == localhost.query( ".capabilities::architecture" )