def test_k8sConfig(self):
os.environ["TEST_SECRET"] = "a secret"
manifest = YamlManifest(manifestScript)
job = Runner(manifest).run(JobOptions(add=True, startTime=1))
assert not job.unexpectedAbort
assert job.status == Status.ok, job.summary()
# print(job.summary())
# print(job.out.getvalue())
# verify secret contents isn't saved in config
self.assertNotIn("a secret", job.out.getvalue())
self.assertNotIn("YSBzZWNyZXQ",
job.out.getvalue()) # base64 of "a secret"
# print (job.out.getvalue())
self.assertIn("<<REDACTED>>", job.out.getvalue())
assert not job.unexpectedAbort
assert job.status == Status.ok, job.summary()
manifest = YamlManifest(job.out.getvalue())
job2 = Runner(manifest).run(
JobOptions(workflow="undeploy", startTime=2))
results = job2.jsonSummary()
assert not job2.unexpectedAbort
assert job2.status == Status.ok, job2.summary()
assert len(results["tasks"]) == 1, results
def test_manifest(self):
path = __file__ + "/../examples/helm-manifest.yaml"
manifest = YamlManifest(path=path)
runner = Runner(manifest)
assert not manifest.lastJob, "expected new manifest"
output = six.StringIO() # so we don't save the file
job = runner.run(JobOptions(add=True, out=output, startTime="test"))
assert not job.unexpectedAbort, job.unexpectedAbort.getStackTrace()
# manifest shouldn't have changed
# print("1", output.getvalue())
manifest2 = YamlManifest(output.getvalue())
# manifest2.statusSummary()
output2 = six.StringIO()
job2 = Runner(manifest2).run(
JobOptions(add=True, out=output2, startTime=1))
# print("2", output2.getvalue())
# print(job2.summary())
assert not job2.unexpectedAbort, job2.unexpectedAbort.getStackTrace()
# should not have found any tasks to run:
assert len(job2.workDone) == 0, job2.workDone
self.maxDiff = None
# self.assertEqual(output.getvalue(), output2.getvalue())
output3 = six.StringIO()
manifest3 = YamlManifest(output2.getvalue())
job3 = Runner(manifest3).run(
JobOptions(workflow="undeploy", out=output3, startTime=2))
# print(output3.getvalue())
# only the chart delete task should have ran as it owns the resources it created
# print(job3.jsonSummary())
assert len(job3.workDone) == 1, job3.jsonSummary()
tasks = list(job3.workDone.values())
assert tasks[0].target.status.name == "absent", tasks[0].target.status
def test_discover(self):
path = __file__ + "/../examples/helm-manifest.yaml"
manifest = YamlManifest(path=path)
runner = Runner(manifest)
assert not manifest.lastJob, "expected new manifest"
output = six.StringIO() # so we don't save the file
job = runner.run(JobOptions(workflow="discover", out=output, startTime=1))
# print(job.summary())
# print("discovered", runner.manifest.tosca.discovered)
# print("discovered manifest", output.getvalue())
assert not job.unexpectedAbort, job.unexpectedAbort.getStackTrace()
baseDir = __file__ + "/../examples/"
manifest2 = YamlManifest(output.getvalue(), path=baseDir)
manifest2.manifest.path = os.path.abspath(
path
) # set the basedir which sets the current working dir
# manifest2.statusSummary()
output2 = six.StringIO()
job2 = Runner(manifest2).run(
JobOptions(workflow="discover", out=output2, startTime=2)
)
# print("2", output2.getvalue())
# print('job2', job2.summary())
assert not job2.unexpectedAbort, job2.unexpectedAbort.getStackTrace()
# print("job", json.dumps(job2.jsonSummary(), indent=2))
# should not have found any tasks to run:
assert len(job2.workDone) == 8, list(job2.workDone)
def test_deploy(self):
# make sure this works
f = urllib.request.urlopen(
"http://localhost:8010/fixtures/helmrepo/index.yaml")
f.close()
runner = Runner(YamlManifest(manifest))
run1 = runner.run(JobOptions(planOnly=True, verbose=3, startTime=1))
mysql_release = runner.manifest.rootResource.findResource(
"mysql_release")
query = ".::.requirements::[.name=host]::.target::name"
res = mysql_release.query(query)
assert res == 'unfurl-helm-unittest'
runner = Runner(YamlManifest(manifest))
run1 = runner.run(JobOptions(dryrun=False, verbose=3, startTime=1))
assert not run1.unexpectedAbort, run1.unexpectedAbort.getStackTrace()
summary = run1.jsonSummary()
# runner.manifest.statusSummary()
# print(summary)
self.assertEqual(
summary["job"],
{
"id": "A01110000000",
"status": "ok",
"total": 4,
"ok": 4,
"error": 0,
"unknown": 0,
"skipped": 0,
"changed": 4,
},
)
assert all(task["targetStatus"] == "ok"
for task in summary["tasks"]), summary["tasks"]
def test_supervisor(self):
cli_runner = CliRunner()
with cli_runner.isolated_filesystem():
src_path = Path(
__file__).parent / "examples" / "supervisor-ensemble.yaml"
path = shutil.copy(src_path, ".")
runner = Runner(YamlManifest(path=path))
try:
job = runner.run(JobOptions(startTime=1, check=True)) # deploy
assert not job.unexpectedAbort, job.unexpectedAbort.get_stack_trace(
)
summary = job.json_summary()
# print(json.dumps(summary, indent=2))
self.assertEqual(
{
"id": "A01110000000",
"status": "ok",
"total": 5,
"ok": 5,
"error": 0,
"unknown": 0,
"skipped": 0,
"changed": 4,
},
summary["job"],
)
time.sleep(0.25)
f = urllib.request.urlopen("http://127.0.0.1:8012/")
expected = b"Directory listing for /"
self.assertIn(expected, f.read())
runner = Runner(YamlManifest(path=path))
job = runner.run(JobOptions(workflow="undeploy", startTime=2))
assert not job.unexpectedAbort, job.unexpectedAbort.get_stack_trace(
)
summary = job.json_summary()
# print(json.dumps(summary, indent=2))
self.assertEqual(
{
"id": "A01120000000",
"status": "ok",
"total": 3,
"ok": 3,
"error": 0,
"unknown": 0,
"skipped": 0,
"changed": 3,
},
summary["job"],
)
finally:
# NOTE: to manually kill: pkill -lf supervisord
if os.path.exists("supervisord/local/supervisord.pid"):
with open("supervisord/local/supervisord.pid") as f:
pid = int(f.read())
print("killing", pid)
os.kill(pid, signal.SIGINT)
def test_ansibleVault(self):
manifest = YamlManifest(manifestDoc, vault=make_vault_lib("a_password"))
outputIp, job = self._runInputAndOutputs(manifest)
vaultString = "server_ip: !vault |\n $ANSIBLE_VAULT;1.1;AES256"
assert vaultString in job.out.getvalue(), job.out.getvalue()
from unfurl.yamlloader import cleartext_yaml
manifest = YamlManifest(manifestDoc, vault=cleartext_yaml.representer.vault)
outputIp, job = self._runInputAndOutputs(manifest)
assert "!vault" not in job.out.getvalue(), job.out.getvalue()
def test_supervisor(self):
cliRunner = CliRunner()
with cliRunner.isolated_filesystem():
runner = Runner(YamlManifest(manifest))
try:
job = runner.run(JobOptions(startTime=1)) # deploy
assert not job.unexpectedAbort, job.unexpectedAbort.getStackTrace(
)
summary = job.jsonSummary()
self.assertEqual(
{
"id": "A01110000000",
"status": "ok",
"total": 4,
"ok": 4,
"error": 0,
"unknown": 0,
"skipped": 0,
"changed": 3,
},
summary["job"],
)
# print(json.dumps(summary, indent=2))
time.sleep(0.25)
f = urllib.request.urlopen("http://127.0.0.1:8012/")
expected = b"Directory listing for /"
self.assertIn(expected, f.read())
runner = Runner(YamlManifest(job.out.getvalue()))
job = runner.run(JobOptions(workflow="undeploy", startTime=2))
assert not job.unexpectedAbort, job.unexpectedAbort.getStackTrace(
)
summary = job.jsonSummary()
# print(json.dumps(summary, indent=2))
self.assertEqual(
{
"id": "A01120000000",
"status": "ok",
"total": 3,
"ok": 3,
"error": 0,
"unknown": 0,
"skipped": 0,
"changed": 3,
},
summary["job"],
)
finally:
if os.path.exists("supervisord/local/supervisord.pid"):
with open("supervisord/local/supervisord.pid") as f:
pid = f.read()
print("killing", pid)
os.kill(pid)
def test_fileRef(self):
simple = ("""
apiVersion: %s
kind: Manifest
spec:
service_template:
topology_template:
node_templates:
test:
type: tosca.nodes.Root
properties:
file:
eval:
file: foo.txt
interfaces:
Standard:
create:
implementation: FileTest
inputs:
path:
eval: file::path
contents:
eval: file::contents
""" % API_VERSION)
cliRunner = CliRunner()
with cliRunner.isolated_filesystem(): # as tmpDir
manifest = YamlManifest(simple, path=".")
runner = Runner(manifest)
output = six.StringIO()
with open("foo.txt", "w") as f:
f.write("test")
job = runner.run(JobOptions(add=True, out=output,
startTime="test"))
task = list(job.workDone.values())[0]
self.assertEqual(task.result.result, "foo.txt")
def test_deploy(self):
# make sure this works
f = urllib.request.urlopen(
"http://localhost:8010/fixtures/helmrepo/index.yaml")
f.close()
runner = Runner(YamlManifest(manifest))
run1 = runner.run(JobOptions(dryrun=False, verbose=3, startTime=1))
assert not run1.unexpectedAbort, run1.unexpectedAbort.getStackTrace()
summary = run1.jsonSummary()
# runner.manifest.statusSummary()
# print(summary)
self.assertEqual(
summary["job"],
{
"id": "A01110000000",
"status": "ok",
"total": 4,
"ok": 4,
"error": 0,
"unknown": 0,
"skipped": 0,
"changed": 4,
},
)
assert all(task["targetStatus"] == "ok"
for task in summary["tasks"]), summary["tasks"]
def test_hasVersion(self):
hasVersion = """
apiVersion: unfurl/v1alpha1
kind: Manifest
spec: {}
"""
assert YamlManifest(hasVersion)
def test_result_template_errors(caplog):
manifest = """\
apiVersion: unfurl/v1alpha1
kind: Ensemble
spec:
service_template:
topology_template:
node_templates:
testNode:
type: tosca.nodes.Root
interfaces:
Standard:
operations:
configure:
implementation:
className: unfurl.configurators.TemplateConfigurator
inputs:
resultTemplate: |
- name: .self
attributes:
outputVar: "{{ SELF.missing }}"
"""
runner = Runner(YamlManifest(manifest))
job = runner.run()
assert not job.unexpectedAbort, job.unexpectedAbort.get_stack_trace()
for record in caplog.records:
if record.levelname == "WARNING":
assert (
record.getMessage() ==
'error processing resultTemplate for testNode: <<Error rendering template: missing attribute or key: "missing">>'
)
break
else:
assert False, "log message not found"
def test_missingConfigurator(self):
manifest = """
apiVersion: unfurl/v1alpha1
kind: Manifest
configurators:
step2:
actions:
install: foo
templates:
base:
configurations:
- step1
- step2
root:
resources:
cloud3:
spec:
templates:
- base
configurations:
+templates/base/configurations:
"""
with self.assertRaises(UnfurlValidationError) as err:
YamlManifest(manifest)
self.assertIn(
str(err.exception.errors),
"""[<ValidationError: "['step1', 'step2'] is not of type 'object'">]""",
)
def get_manifest(self,
path: Optional[str]=None,
skip_validation: bool=False
) -> "YamlManifest":
from .yamlmanifest import YamlManifest
if path and path != self.manifestPath:
# share projects and ensembles
localEnv = LocalEnv(path, parent=self)
return localEnv.get_manifest()
else:
assert self.manifestPath
manifest = self._manifests.get(self.manifestPath)
if not manifest:
# should load vault ids from context
project = self.project or self.homeProject
if project:
vault = project.make_vault_lib(self.manifest_context_name)
if vault:
self.logger.info(
"Vault password found, configuring vault ids: %s",
[s[0] for s in vault.secrets],
)
else:
vault = None
if not vault:
msg = "No vault password found"
if self.manifest_context_name:
msg += f" for environment {self.manifest_context_name}"
self.logger.debug(msg)
manifest = YamlManifest(
localEnv=self, vault=vault, skip_validation=skip_validation
)
self._manifests[self.manifestPath] = manifest
return manifest
def test_login(self):
"""
test that runner figures out the proper tasks to run
"""
import docker
client = docker.from_env()
assert client, "docker not installed?"
runner = Runner(YamlManifest(manifest))
# pickled = pickle.dumps(runner.manifest, -1)
# manifest2 = pickle.loads(pickled)
run1 = runner.run(JobOptions(instance="test1"))
assert len(run1.workDone) == 1, run1.workDone
tasks = list(run1.workDone.values())
# docker login will fail because user doesn't exist:
assert tasks[0].status.name == "error", tasks[0].status
self.assertIn("401 Client Error",
tasks[0].result.result.get("msg", ""))
# but the repository and image path will have been created
self.assertEqual(
tasks[0].result.outputs.get("image_path"),
"index.docker.io/repo/image",
)
registry = tasks[0].result.outputs.get("registry")
assert registry and isinstance(registry,
toscaparser.repositories.Repository)
assert not run1.unexpectedAbort, run1.unexpectedAbort.get_stack_trace()
def test_missing_type_is_handled_by_unfurl(self):
ensemble = """
apiVersion: unfurl/v1alpha1
kind: Ensemble
configurations:
create:
implementation:
className: unfurl.configurators.shell.ShellConfigurator
inputs:
command: echo hello
spec:
service_template:
topology_template:
node_templates:
test_node:
# type: tosca.nodes.Root
interfaces:
Standard:
+/configurations:
"""
with self.assertRaises(UnfurlValidationError) as err:
YamlManifest(ensemble)
assert (
'MissingRequiredFieldError: Template "test_node" is missing required field "type"'
in str(err.exception)
)
def test_lifecycle_relationships(self):
manifest = YamlManifest(ENSEMBLE_WITH_RELATIONSHIPS)
steps = list(DEFAULT_STEPS)
# steps[0] = Step("check", Status.ok)
jobs = lifecycle(manifest, steps)
for job in jobs:
assert job.status == Status.ok, job.workflow
def test_ansible(self):
"""
Run ansible command on a (mock) remote instance.
"""
try:
oldTmpDir = os.environ["UNFURL_TMPDIR"]
runner = CliRunner()
with runner.isolated_filesystem() as tempDir:
os.environ["UNFURL_TMPDIR"] = tempDir
path = __file__ + "/../examples/ansible-manifest.yaml"
manifest = YamlManifest(path=path)
runner = Runner(manifest)
output = six.StringIO() # so we don't save the file
job = runner.run(
JobOptions(
workflow="run",
host="www.example.com",
# this instance doesn't exist so warning is output
instance="www.example.com",
cmdline=["echo", "foo"],
out=output,
)
)
assert not job.unexpectedAbort, job.unexpectedAbort.getStackTrace()
# verify we wrote out the correct ansible inventory file
try:
from pathlib import Path
p = Path(os.environ["UNFURL_TMPDIR"])
files = list(p.glob("**/*-inventory.yaml"))
self.assertEqual(len(files), 1, files)
inventory = files[-1]
expectedInventory = """all:
hosts:
www.example.com:
ansible_port: 22
ansible_connection: local
ansible_user: ubuntu
ansible_pipelining: yes
ansible_private_key_file: ~/.ssh/example-key.pem
vars: {}
children:
example_group:
hosts: {}
vars:
var_for_ansible_playbook: test
children: {}
"""
with inventory.open() as f:
self.assertEqual(f.read(), expectedInventory)
except ImportError:
pass # skip on 2.7
finally:
os.environ["UNFURL_TMPDIR"] = oldTmpDir
tasks = list(job.workDone.values())
self.assertEqual(len(tasks), 1)
assert "stdout" in tasks[0].result.result, tasks[0].result.result
self.assertEqual(tasks[0].result.result["stdout"], "foo")
def test_lifecycle_exclusive(self):
manifest = YamlManifest(
ENSEMBLE_ROUTE53.replace("exclusive: false", "exclusive: true"))
jobs = lifecycle(manifest)
for job in jobs:
assert job.rootResource.find_resource(
"test_node").attributes["exclusive"]
assert job.status == Status.ok, job.workflow
def test_timeout_with_ensemble(self):
runner = Runner(YamlManifest(ENSEMBLE_TIMEOUT))
start_time = datetime.now()
job = runner.run(JobOptions(instance="test_node"))
delta = datetime.now() - start_time
assert job.status == Status.error
assert delta < timedelta(seconds=2), delta - timedelta(seconds=2)
def test_schemas(self):
basedir = os.path.join(os.path.dirname(__file__), "..", "docs",
"examples")
assert LocalConfig(os.path.join(basedir, "unfurl.yaml"))
assert YamlManifest(path=os.path.join(basedir, "ensemble.yaml"))
assert YamlConfig(
path=os.path.join(basedir, "job.yaml"),
schema=os.path.join(_basepath, "changelog-schema.json"),
)
def test_validateVersion(self):
badVersion = """
apiVersion: 2
kind: Manifest
spec: {}
"""
with self.assertRaises(UnfurlError) as err:
YamlManifest(badVersion)
self.assertIn("apiVersion", str(err.exception))
missingVersion = """
spec: {}
"""
with self.assertRaises(UnfurlError) as err:
YamlManifest(missingVersion)
self.assertIn(
"'apiVersion' is a required property", str(err.exception)
) # , <ValidationError: "'kind' is a required property">]''')
def verifyRoundtrip(self, original, jobOptions):
if jobOptions.startTime:
jobOptions.startTime += 1
job = Runner(YamlManifest(original)).run(jobOptions)
# should not need to run any tasks
assert len(job.workDone) == 0, job.workDone
self.maxDiff = None
self.assertEqual(original, job.out.getvalue())
def test_run_without_dry_run(self, command, dryrun):
ensemble = ENSEMBLE_DRY_RUN.format(command=command, dryrun=dryrun)
runner = Runner(YamlManifest(ensemble))
job = runner.run(JobOptions(instance="test_node", dryrun=False))
assert job.status == Status.ok
task = list(job.workDone.values())[0]
cmd = task.result.result["cmd"].strip()
assert cmd == "echo hello world"
def test_manifest(self):
simple = (
"""
apiVersion: %s
kind: Manifest
spec:
instances:
anInstance:
# template: foo
interfaces:
Standard:
operations:
configure:
implementation: TestSubtask
inputs: {}
"""
% API_VERSION
)
manifest = YamlManifest(simple)
runner = Runner(manifest)
self.assertEqual(runner.taskCount, 0)
output = six.StringIO()
job = runner.run(JobOptions(add=True, out=output, startTime="test"))
assert not job.unexpectedAbort, job.unexpectedAbort.getStackTrace()
# workDone includes subtasks
assert len(job.workDone) == 2, job.workDone
# manifest shouldn't have changed
manifest2 = YamlManifest(output.getvalue())
lock = manifest2.manifest.config["lock"]
assert "runtime" in lock and len(lock["repositories"]) == 3
self.assertEqual(
manifest2.lastJob["summary"],
"2 tasks (2 changed, 2 ok, 0 failed, 0 unknown, 0 skipped)",
)
output2 = six.StringIO()
job2 = Runner(manifest2).run(JobOptions(add=True, out=output2))
assert not job2.unexpectedAbort, job2.unexpectedAbort.getStackTrace()
# should not find any tasks to run
assert len(job2.workDone) == 0, job2.workDone
self.maxDiff = None
self.assertEqual(output.getvalue(), output2.getvalue())
def test_exclusive(self, manager_sync):
runner = Runner(YamlManifest(ENSEMBLE_EXCLUSIVE))
job = runner.run(JobOptions(workflow="deploy"))
assert job.status == Status.ok
node = job.rootResource.find_resource("test_node")
# records are replaced by instance
assert len(node.attributes["zone"]) == 1
assert manager_sync.called
def test_error_if_dry_run_not_defined_for_task(self):
ensemble = ENSEMBLE_DRY_RUN.format(command="command: echo hello world",
dryrun="")
runner = Runner(YamlManifest(ensemble))
job = runner.run(JobOptions(instance="test_node", dryrun=True))
task = list(job.workDone.values())[0]
assert job.status == Status.error
assert task.result.result == "could not run: dry run not supported"
def test_check(self):
runner = Runner(YamlManifest(ENSEMBLE_ROUTE53))
runner.run(JobOptions(workflow="deploy"))
job = runner.run(JobOptions(workflow="check"))
assert job.status == Status.ok
task = list(job.workDone.values())[0]
# this means that dns records were correctly set during deploy:
assert task.target_status == Status.ok
assert task.result.result == "DNS records in sync"
def test_shell(self):
"""
test that runner figures out the proper tasks to run
"""
runner = Runner(YamlManifest(self.ensemble))
job = runner.run(JobOptions(instance="test1"))
assert len(job.workDone) == 1, job.workDone
node = runner.manifest.get_root_resource().find_resource("test1")
assert node.attributes["stdout"] == "helloworld"
assert not job.unexpectedAbort, job.unexpectedAbort.get_stack_trace()
def test_configurator(self):
"""
test that runner figures out the proper tasks to run
"""
runner = Runner(YamlManifest(manifest))
run1 = runner.run(JobOptions(resource="test1"))
assert not run1.unexpectedAbort, run1.unexpectedAbort.getStackTrace()
assert len(run1.workDone) == 1, run1.workDone
result = list(run1.workDone.values())[0].result
self.assertEqual(result.outputs, {"fact1": "test1", "fact2": "test"})
self.assertEqual(result.result.get("stdout"), sys.executable)
assert run1.status == Status.ok, run1.summary()
def test_neededTasks(self):
"""
test that runner figures out the proper tasks to run
"""
runner = Runner(YamlManifest(manifest))
test1 = runner.manifest.get_root_resource().find_resource("test1")
assert test1
# missing = runner.manifest.spec[0].findInvalidPreconditions(test1)
# assert not missing, missing
run1 = runner.run(JobOptions(instance="test1"))
assert not run1.unexpectedAbort, run1.unexpectedAbort.get_stack_trace()
assert len(run1.workDone) == 1, run1.summary()