def test_render_section_invalid_template():
template_string = """
{% for %}
"""
state = {"foo": 2}
section = {"template": template_string}
with pytest.raises(ValidationError):
parsing.render_section(section, state=state)
def test_render_section_dict_no_template():
section = {"foo": "{{ state['bar'] }}", "fizz": "{{ json.loads('true') }}"}
state = {"bar": 24}
rendered_section = parsing.render_section(section, state=state)
assert rendered_section == {"foo": 24, "fizz": True}
def test_render_section_string():
template_string = "{{ state['foo'] }}"
state = {"foo": 24}
rendered_section = parsing.render_section(template_string, state=state)
assert rendered_section == 24
def run_asserts(asserts: List[Assert], state: dict, hostname: str,
seconds_between_asserts: float) -> Statuses:
"""
Run asserts assigned to host
Args:
asserts: List of asserts assigned to host
state: Test state to pass to templates
hostname: Host name to send assert data to
seconds_between_asserts: Time to wait in between running each assert
Returns:
Statuses of all asserts run by host
"""
results: Statuses = {}
for i, asrt in enumerate(asserts):
rendered_assert: Assert = render_section(asrt, state)
assert_name = rendered_assert.get("name")
executions_per_cycle = rendered_assert.get("executionsPerCycle", 1)
assert_results: List[AssertResult] = []
if rendered_assert["type"] == "NullAssert":
for _ in range(executions_per_cycle):
# NOTE: possibly add template free way of computing passed
assert_results.append(
AssertResult(
passed=rendered_assert.get("passed", False),
actual=rendered_assert.get("actual", ""),
expected=rendered_assert.get("expected", ""),
description=rendered_assert.get("description", ""),
))
else:
assert ("params" in rendered_assert
), f"Assert {assert_name} is missing property 'params'"
for _ in range(executions_per_cycle):
assert_results.append(send_assert(hostname, rendered_assert))
save_assert_versions = rendered_assert.get("storeVersions", True)
if not save_assert_versions:
results[assert_name] = assert_results[-1]
else:
results[assert_name] = assert_results
if i != len(asserts) - 1:
# Only wait if there is another assert
time.sleep(seconds_between_asserts)
return results
def test_render_section_kwargs():
template_string = """
params:
value: {{ state['foo'] + results['bar'] }}
"""
state = {"foo": 2}
results = {"bar": 4}
section = {"template": template_string}
rendered_section = parsing.render_section(section,
state=state,
results=results)
assert rendered_section["params"]["value"] == 6
def test_render_section_multiple_parts():
template_string = """
image: {{ state['foo'] }}
volumes:
- name: {{ state['bar'] }}
path: bar
"""
section = {"name": "some name", "template": template_string}
state = {"foo": "fizz", "bar": "buzz"}
rendered_section = parsing.render_section(section, state=state)
assert rendered_section["image"] == "fizz"
assert rendered_section["volumes"][0]["name"] == "buzz"
def test_render_section():
template_string = """
params:
{% set ids = [] %}
{% for member in state['add_members']['actions']['POST0']['results'] %}
{% do ids.append(member['body']['id']) %}
{% endfor %}
query: "update members set name='jeff2' where id in ({{ ids|join(',') }})"
"""
section = {"type": "SQLQuery", "template": template_string}
state = {
"add_members": {
"actions": {
"POST0": {
"results": [
{
"body": {
"id": 1
}
},
{
"body": {
"id": 2
}
},
]
}
}
}
}
rendered_section = parsing.render_section(section, state=state)
assert rendered_section["type"] == "SQLQuery"
assert rendered_section["params"] == {
"query": "update members set name='jeff2' where id in (1,2)"
}
def test_render_section_not_found():
section = {"foo": "{{ bar }}"}
rendered_section = parsing.render_section(section, state={})
assert rendered_section == {"foo": None}
def closure(state):
try:
rendered_test_config: TestConfig = render_section(
test_config, state)
image = runner_to_image(rendered_test_config.get(
"runner")) or rendered_test_config.get("image")
assert image is not None, "Must specify a valid 'runner' or 'image'"
env = config_to_runner_env(
render_section(rendered_test_config.get("config", {}), state))
runners = []
for _ in range(rendered_test_config.get("runnerCount", 1)):
runner = create_runner_fn(
image,
env,
run_id,
volumes=rendered_test_config.get("volumes"))
runners.append(runner)
try:
new_state = run_test_with_timeout(
test_config=rendered_test_config,
incoming_state=state,
hostnames=[
get_runner_hostname_fn(runner) for runner in runners
],
duration=rendered_test_config.get("timeout", 15),
)
except (AssertionError, ValueError, TypeError,
RuntimeError) as err:
# NOTE: May need to fine tune exception types
LOGGER.error("Error running test %s: %s",
test_config["name"],
err,
exc_info=True)
for runner in runners:
remove_runner_fn(runner)
new_state = {
test_config["name"]: {
"summary":
TestSummary(
description=rendered_test_config.get(
"description"),
error=str(err),
completed_cycles=0,
remaining_asserts=[],
duration=0,
filename=rendered_test_config.get("filename"),
)
}
}
for runner in runners:
remove_runner_fn(runner)
except (AssertionError, ValueError, TypeError, RuntimeError) as err:
LOGGER.error("Error creating test %s: %s",
test_config["name"],
err,
exc_info=True)
new_state = {
test_config["name"]: {
"summary":
TestSummary(
description=test_config.get("description"),
error=str(err),
completed_cycles=0,
remaining_asserts=[],
duration=0,
filename=test_config.get("filename"),
)
}
}
return {**state, **new_state}
def run_actions(
actions: List[Action], state: dict, hostname: str, seconds_between_actions: float
) -> ActionsData:
"""
Runs a list of actions assigned to a single runner
Args:
actions: List of actions to run
state: Incoming state to use in rendering actions
hostname: Address of runner
seconds_between_actions: Seconds to wait between running next action in list
Returns:
ActionsData per action provided
"""
def infinite_defaultdict():
return defaultdict(infinite_defaultdict)
data: ActionsData = OrderedDict(
(action["name"], infinite_defaultdict()) for action in actions
)
with get_action_sender(hostname) as send_action:
for i, action in enumerate(actions):
rendered_action: Action = render_section(action, state)
action_name = rendered_action["name"]
assert (
"params" in rendered_action
), f"Action {action_name} is missing property 'params'"
executions_per_cycle: int = rendered_action.get("executionsPerCycle", 1)
action_results: List[ActionResult] = []
# assert_results: Statuses = defaultdict(list)
assert_results: Statuses = OrderedDict(
(asrt["name"], []) for asrt in rendered_action.get("asserts", [])
)
for _ in range(executions_per_cycle):
execution_output: ActionResult = send_action(rendered_action)
action_results.append(execution_output)
for asrt in get_remaining_asserts(
rendered_action.get("asserts", []), assert_results
):
rendered_assert: Assert = render_section(
section=asrt, state=state, result=execution_output
)
assert_name = asrt["name"]
store_assert_versions = rendered_assert.get("storeVersions", True)
assert_result = run_assert_from_action_result(
rendered_assert, execution_output
)
if store_assert_versions:
assert_results[assert_name].append(assert_result)
else:
assert_results[assert_name] = assert_result
time.sleep(rendered_action.get("secondsBetweenExecutions", 0))
store_action_versions = rendered_action.get("storeVersions", True)
if not store_action_versions and action_results:
data[action_name]["results"] = action_results[-1]
else:
data[action_name]["results"] = action_results
data[action_name]["asserts"] = assert_results
for output in rendered_action.get("outputs", []):
rendered_output: Output = render_section(
section=output, state=state, results=action_results
)
assert (
"name" in rendered_output
), "Output section must have parameter 'name'"
assert (
"value" in rendered_output
), "Output section must have parameter 'value'"
# NOTE: support updating outputs in globals section?
store_output_versions = rendered_output.get("storeVersions", False)
if not store_output_versions:
data[action_name]["outputs"][
rendered_output["name"]
] = rendered_output["value"]
else:
data[action_name]["outputs"][rendered_output["name"]] = [
rendered_output["value"]
]
if i != len(actions) - 1:
# Only wait if there is another action
time.sleep(seconds_between_actions)
return data
def closure(state):
try:
# TODO: break out docker specific sections
rendered_test_config = render_section(test_config, state)
image = runner_to_image(rendered_test_config.get(
"runner")) or rendered_test_config.get("image")
assert image is not None, "Must specify a valid 'runner' or 'image'"
env = config_to_runner_env(
render_section(rendered_test_config.get("config", {}), state))
# NOTE: client may need more config options
client: docker.DockerClient = docker.from_env()
containers = []
for _ in range(test_config.get("runnerCount", 1)):
container = create_docker_container(client, image, env, run_id)
LOGGER.info("successfully created container %s",
container.name)
containers.append(container)
try:
new_state = run_test_with_timeout(
test_config=rendered_test_config,
incoming_state=state,
hostnames=[
f"{container.name}:50051" for container in containers
],
duration=rendered_test_config.get("timeout", 15),
)
except (AssertionError, ValueError, TypeError,
RuntimeError) as err:
# NOTE: May need to fine tune exception types
LOGGER.error("Error running test %s: %s",
test_config["name"],
err,
exc_info=True)
for container in containers:
LOGGER.debug("Stopping runner %s", container.name)
container.stop(timeout=3)
new_state = {
test_config["name"]: {
"summary":
TestSummary(
description=test_config.get("description"),
error=str(err),
completed_cycles=0,
remaining_asserts=[],
duration=0,
)
}
}
for container in containers:
LOGGER.debug("Stopping runner %s", container.name)
container.stop(timeout=3)
except (AssertionError, ValueError, TypeError, RuntimeError) as err:
LOGGER.error("Error creating test %s: %s",
test_config["name"],
err,
exc_info=True)
new_state = {
test_config["name"]: {
"summary":
TestSummary(
description=test_config.get("description"),
error=str(err),
completed_cycles=0,
remaining_asserts=[],
duration=0,
)
}
}
return {**state, **new_state}