def test_run_asserts(mock_send_assert):
mock_send_assert.return_value = AssertResult(
passed=True, actual="foo", expected="foo", description="good"
)
test_asserts = [
{"name": "A", "type": "SQLAssert", "params": {}, "executionsPerCycle": 2},
{"name": "B", "type": "NullAssert", "passed": True, "params": {}},
{"name": "C", "type": "NullAssert", "params": {}},
]
statuses = asserts.run_asserts(test_asserts, {}, "", 0)
assert statuses["A"] == [
AssertResult(passed=True, actual="foo", expected="foo", description="good"),
AssertResult(passed=True, actual="foo", expected="foo", description="good"),
]
assert statuses["B"] == [
AssertResult(passed=True, actual="", expected="", description="")
]
assert statuses["C"] == [
AssertResult(passed=False, actual="", expected="", description="")
]
def test_asserts_non_versioned(mock_send_assert):
mock_send_assert.return_value = AssertResult(
passed=True, actual="foo", expected="foo", description="good"
)
test_asserts = [
{"name": "A", "type": "SQLAssert", "storeVersions": False, "params": {}},
{
"name": "B",
"type": "NullAssert",
"passed": True,
"storeVersions": False,
"params": {},
},
{"name": "C", "type": "NullAssert", "params": {}},
]
statuses = asserts.run_asserts(test_asserts, {}, "", 0)
assert statuses["A"] == AssertResult(
passed=True, actual="foo", expected="foo", description="good"
)
assert statuses["B"] == AssertResult(
passed=True, actual="", expected="", description=""
)
assert statuses["C"] == [
AssertResult(passed=False, actual="", expected="", description="")
]
def test_run_asserts_series_one_host_multiple_asserts(run_asserts_mock: Mock):
run_asserts_mock.return_value = {
"A":
[AssertResult(passed=True, actual="", expected="", description="")],
"B":
[AssertResult(passed=False, actual="", expected="", description="")],
}
test_state = defaultdict(dict)
test_asserts = [
{
"name": "A",
"type": "SQLAssert"
},
{
"name": "B",
"type": "SQLAssert"
},
]
results = testing.run_asserts_series(
asserts=test_asserts,
state=test_state,
test_name="foo",
hostnames=["alpha"],
seconds_between_asserts=0,
)
assert results == {
"A":
[AssertResult(passed=True, actual="", expected="", description="")],
"B":
[AssertResult(passed=False, actual="", expected="", description="")],
}
def run_assert(assert_type: str, params: AssertParams) -> AssertResult:
action_type = params["actionType"]
action_params = params["actionParams"]
expected = params["expected"]
assert_options = params.get("assertOptions", {})
action_result = run_action(action_type, action_params)
if assert_type == "ResponseAssert":
actual = action_result["response"]
passed, description = assert_element(expected, actual,
**assert_options)
elif assert_type == "MetadataAssert":
actual = action_result["metadata"]
passed, description = assert_element(expected, actual,
**assert_options)
elif assert_type == "ErrorAssert":
actual = action_result["error"]
passed, description = assert_element(expected, actual,
**assert_options)
else:
raise ValueError(f"Assert type {assert_type} is invalid")
return AssertResult(
actual=json.dumps(actual),
expected=json.dumps(expected),
passed=passed,
description=json.dumps(description),
)
def test_get_remaining_asserts():
test_asserts = [
{"name": "A"},
{"type": "B", "name": "B0"},
{"type": "B", "name": "B1"},
{"type": "C", "name": "C0"},
]
statuses = {
"A": [
AssertResult(passed=False, actual="", expected="", description=""),
AssertResult(passed=True, actual="", expected="", description=""),
],
"B0": [
AssertResult(passed=True, actual="", expected="", description=""),
AssertResult(passed=True, actual="", expected="", description=""),
],
"B1": [
AssertResult(passed=False, actual="", expected="", description=""),
AssertResult(passed=False, actual="", expected="", description=""),
],
"C0": [AssertResult(passed=False, actual="", expected="", description="")],
}
remaining_asserts = asserts.get_remaining_asserts(test_asserts, statuses)
assert len(remaining_asserts) == 2
assert {"type": "B", "name": "B1"} in remaining_asserts
assert {"type": "C", "name": "C0"} in remaining_asserts
def test_continue_running_asserts_passed_limited_second_run_no_remaining():
test_asserts = [{"type": "RESTAssert"}]
assert_statuses = {
"RESTAssert0":
[AssertResult(passed=True, actual="", expected="", description="")],
}
assert not testing.continue_running(asserts=test_asserts,
remaining_cycles=0,
assert_statuses=assert_statuses)
def test_continue_running_asserts_failed_unlimited_second_run():
test_asserts = [{"type": "RESTAssert"}]
assert_statuses = {
"RESTAssert0":
[AssertResult(passed=False, actual="", expected="", description="")],
}
assert testing.continue_running(asserts=test_asserts,
remaining_cycles=-2,
assert_statuses=assert_statuses)
def test_run_asserts_series_multiple_hosts_one_assert(run_asserts_mock: Mock):
run_asserts_mock.side_effect = [{
"A":
[AssertResult(passed=True, actual="", expected="", description="")]
}]
test_state = defaultdict(dict)
test_asserts = [{"name": "A", "type": "SQLAssert"}]
results = testing.run_asserts_series(
asserts=test_asserts,
state=test_state,
test_name="foo",
hostnames=["alpha", "bravo"],
seconds_between_asserts=0,
)
assert results == {
"A":
[AssertResult(passed=True, actual="", expected="", description="")]
}
def run_asserts(asserts: List[Assert], state: dict, hostname: str,
seconds_between_asserts: float) -> Statuses:
"""
Run asserts assigned to host
Args:
asserts: List of asserts assigned to host
state: Test state to pass to templates
hostname: Host name to send assert data to
seconds_between_asserts: Time to wait in between running each assert
Returns:
Statuses of all asserts run by host
"""
results: Statuses = {}
for i, asrt in enumerate(asserts):
rendered_assert: Assert = render_section(asrt, state)
assert_name = rendered_assert.get("name")
executions_per_cycle = rendered_assert.get("executionsPerCycle", 1)
assert_results: List[AssertResult] = []
if rendered_assert["type"] == "NullAssert":
for _ in range(executions_per_cycle):
# NOTE: possibly add template free way of computing passed
assert_results.append(
AssertResult(
passed=rendered_assert.get("passed", False),
actual=rendered_assert.get("actual", ""),
expected=rendered_assert.get("expected", ""),
description=rendered_assert.get("description", ""),
))
else:
assert ("params" in rendered_assert
), f"Assert {assert_name} is missing property 'params'"
for _ in range(executions_per_cycle):
assert_results.append(send_assert(hostname, rendered_assert))
save_assert_versions = rendered_assert.get("storeVersions", True)
if not save_assert_versions:
results[assert_name] = assert_results[-1]
else:
results[assert_name] = assert_results
if i != len(asserts) - 1:
# Only wait if there is another assert
time.sleep(seconds_between_asserts)
return results
def run_assert(assert_type: str, params: AssertParams) -> AssertResult:
if assert_type == "FindMessage":
messages = run_action("Receive", params["actionParams"])
for message in messages["messages_received"]:
if assert_dicts(params["expected"], message):
return AssertResult(
actual=str(message),
expected=str(params["expected"]),
passed=True,
description="passed",
)
return AssertResult(
actual=None,
expected=str(params["expected"]),
passed=False,
description=f"No message found matching {params['expected']}",
)
raise ValueError(f"Assert type {assert_type} is invalid")
def call(asrt: dict):
request = runner_pb2.AssertRequest(type=asrt["type"],
params=json.dumps(
asrt["params"]))
try:
response: runner_pb2.AssertReply = stub.Assert(request)
return AssertResult(
passed=response.passed,
actual=response.actual,
expected=response.expected,
description=response.description,
)
except grpc.RpcError as err:
LOGGER.warning("Received %s during send_assert: %s",
err.code(), err)
return AssertResult(passed=False,
actual=None,
expected=None,
description=err.details())
def run_assert(assert_type: str, params: AssertParams) -> AssertResult:
params_problems = assert_params_problems(params)
if params_problems:
raise ValueError(f"Params invalid: {', '.join(params_problems)}")
action_response = run_action(params["method"], params["actionParams"])
expected = params["expected"]
print(action_response)
if assert_type == "ContainsRows":
# NOTE: possibly validate that expected/action response has rows
expected_rows = expected.get("rows", [])
actual_rows = action_response.get("rows", [])
passed, description = contains_rows(expected_rows, actual_rows)
return AssertResult(
passed=passed,
expected=str(expected_rows),
actual=str(actual_rows),
description=description,
)
elif assert_type == "EqualsRows":
expected_rows = expected.get("rows", [])
actual_rows = action_response.get("rows", [])
passed, description = equals_rows(expected_rows, actual_rows)
return AssertResult(
passed=passed,
expected=str(expected_rows),
actual=str(actual_rows),
description=description,
)
else:
raise ValueError(f"Assert type {assert_type} is invalid")
def test_run_asserts_parallel(run_asserts_mock: Mock):
run_asserts_mock.return_value = {
"A":
[AssertResult(passed=True, actual="", expected="", description="")]
}
test_state = defaultdict(dict)
test_asserts = [{"name": "A", "type": "SQLAssert"}]
results = testing.run_asserts_parallel(
asserts=test_asserts,
state=test_state,
test_name="foo",
hostnames=["alpha", "bravo"],
seconds_between_asserts=0,
)
assert results == {
"A": [
AssertResult(passed=True, actual="", expected="", description=""),
AssertResult(passed=True, actual="", expected="", description=""),
]
}
def test_run_asserts_negate(mock_get_assert_sender):
mock_get_assert_sender.return_value.__enter__.return_value.return_value = (
AssertResult(passed=False,
actual="foo",
expected="foo",
description="bad"))
test_asserts = [{
"name": "A",
"type": "SQLAssert",
"params": {},
"negate": True
}]
statuses = asserts.run_asserts(test_asserts, {}, "", 0)
assert statuses["A"] == [
AssertResult(
passed=True,
actual="foo",
expected="foo",
description="passed; negated: bad",
),
]
def run_assert_from_action_result(asrt: Assert, action_result: ActionResult):
if asrt.get("type") == "NullAssert":
assert_result = AssertResult(
passed=asrt.get("passed", False),
actual=asrt.get("actual", ""),
expected=asrt.get("expected", ""),
description=asrt.get("description", ""),
)
else:
assert (
"expected" in asrt
), f"Assert {asrt.get('name')} is missing property 'expected'"
actual = asrt.get("actual", action_result)
expected = asrt.get("expected")
passed, description = assert_element(
expected, actual, **asrt.get("assertOptions", {})
)
if asrt.get("negate", False):
passed = not passed
if passed:
description = f"passed; negated: {description}"
else:
description = f"expected not {expected}"
assert_result = AssertResult(
passed=passed,
actual=actual,
expected=expected,
description=description,
)
return assert_result
def send_assert(runner_address: str, asrt: dict) -> AssertResult:
with grpc.insecure_channel(runner_address) as channel:
stub = runner_pb2_grpc.RunnerStub(channel)
request = runner_pb2.AssertRequest(type=asrt["type"],
params=json.dumps(
asrt["params"]).encode("utf-8"))
try:
response: runner_pb2.AssertReply = stub.Assert(request)
return AssertResult(
passed=response.passed,
actual=response.actual,
expected=response.expected,
description=response.description,
)
except grpc.RpcError as err:
LOGGER.warning("Received %s during send_assert: %s", err.code(),
err)
return AssertResult(passed=False,
actual=None,
expected=None,
description=err.details())
def test_continue_running_asserts_failed_limited_second_run():
actions = []
asserts = [{"type": "RESTAssert"}]
assert_statuses = {
"RESTAssert0":
[AssertResult(passed=False, actual="", expected="", description="")],
}
assert not testing.continue_running(
actions=actions,
asserts=asserts,
remaining_cycles=0,
actions_data={},
assert_statuses=assert_statuses,
)
def run_assert(assert_type: str, params: AssertParams) -> AssertResult:
params_problems = assert_params_problems(params)
if params_problems:
raise ValueError(f"Params invalid: {', '.join(params_problems)}")
action_response = run_action(params["method"], params["actionParams"])
expected = params["expected"]
if assert_type == "Headers":
actual = action_response["headers"]
passed, description = assert_dicts(
expected=expected,
actual=actual,
all_required=params.get("allRequired", False),
)
elif assert_type == "JSON":
actual = action_response["body"]
passed, description = assert_dicts(
expected=expected,
actual=actual,
all_required=params.get("allRequired", False),
)
elif assert_type == "StatusCode":
actual = action_response["status_code"]
passed = expected == actual
if not passed:
description = f"expected status code {expected}, got {actual}"
else:
description = "passed"
else:
raise ValueError(f"Assert type {assert_type} is invalid")
return AssertResult(
passed=passed,
expected=str(expected),
actual=str(actual),
description=description,
)
def test_continue_running_actions_with_asserts_failed_limited_second_run():
actions = [Action(name="bar", asserts=[Assert(name="foo")])]
asserts = []
actions_data = {
"bar": {
"asserts": {
"foo":
AssertResult(passed=False,
actual="",
expected="",
description="")
}
}
}
assert not testing.continue_running(
actions=actions,
asserts=asserts,
remaining_cycles=0,
actions_data=actions_data,
assert_statuses={},
)
def run_assert(assert_type: str, params: AssertParams) -> AssertResult:
if assert_type == "Exists":
assert "expected" in params, "'expected' must be specified for assert 'Exists'"
assert (
"path" in params or "actionParams" in params
), "'path' or 'actionParams' must be specified for assert 'Exists'"
action_params = params.get("actionParams", {"path": params["path"]})
action_result: ExistsResponse = run_action("read", action_params)
passed = action_result["exists"] == params["expected"]
description = "passed"
if not passed:
description = (
f"expected {params['expected']} but got {action_result['exists']}"
)
return AssertResult(
actual=str(action_result["exists"]),
expected=params["expected"],
passed=passed,
description=description,
)
elif assert_type == "ContentsEqual":
assert ("expected" in params
), "'expected' must be specified for assert 'ContentsEqual'"
assert (
"path" in params or "actionParams" in params
), "'path' or 'actionParams' must be specified for assert 'ContentsEqual'"
action_params = params.get("actionParams", {"path": params["path"]})
action_result: ReadResponse = run_action("read", action_params)
passed, description = assert_strings(params["expected"],
action_result["contents"],
match=False)
return AssertResult(
actual=action_result["contents"],
expected=params["expected"],
passed=passed,
description=description,
)
elif assert_type == "ContentsMatch":
assert ("expected" in params
), "'expected' must be specified for assert 'ContentsMatch'"
assert (
"path" in params or "actionParams" in params
), "'path' or 'actionParams' must be specified for assert 'ContentsMatch'"
action_params = params.get("actionParams", {"path": params["path"]})
action_result: ReadResponse = run_action("read", action_params)
passed, description = assert_strings(params["expected"],
action_result["contents"],
match=True)
return AssertResult(
actual=action_result["contents"],
expected=params["expected"],
passed=passed,
description=description,
)
elif assert_type == "FilesEqual":
assert ("expected" in params
), "'expected' must be specified for assert 'ContentsMatch'"
assert (
"path" in params or "actionParams" in params
), "'path' or 'actionParams' must be specified for assert 'FilesEqual'"
# Use action params if specified, otherwise use provided path
source_path = params.get("actionParams",
{}).get("sourcePath", params["path"])
destination_path = params.get("actionParams",
{}).get("destinationPath",
f"/tmp/{uuid4()}")
action_params = params.get(
"actionParams",
{
"sourcePath": source_path,
"destinationPath": destination_path
},
)
run_action("get", action_params)
passed = cmp(params["expected"], destination_path)
description = "passed"
if not passed:
description = (
f"File contents in {params['expected']} and {source_path} do not match"
)
return AssertResult(
actual=source_path,
expected=params["expected"],
passed=passed,
description=description,
)
else:
raise ValueError(f"Assert type {assert_type} is invalid")
def test_run_tests_limited_cycles(run_asserts_series_mock: Mock):
run_asserts_series_mock.side_effect = [
{
"A": [
AssertResult(passed=False,
actual="",
expected="",
description="")
]
},
{
"A": [
AssertResult(passed=False,
actual="",
expected="",
description=""),
AssertResult(passed=False,
actual="",
expected="",
description=""),
]
},
{
"A": [
AssertResult(passed=False,
actual="",
expected="",
description=""),
AssertResult(passed=False,
actual="",
expected="",
description=""),
AssertResult(passed=False,
actual="",
expected="",
description=""),
]
},
]
test_config = {
"name": "some_test_name",
"cycles": 2,
"secondsBetweenCycles": 0,
"asserts": [{
"name": "A",
"type": "SQLAssert",
"params": {}
}],
"filename": "test.foo.yaml",
}
end_state = testing.run_test(test_config=test_config,
incoming_state={},
hostnames=["alpha"])
assert run_asserts_series_mock.call_count == 2
assert end_state == {
"some_test_name": {
"asserts": {
"A": [
AssertResult(passed=False,
actual="",
expected="",
description=""),
AssertResult(passed=False,
actual="",
expected="",
description=""),
]
},
"summary": {
"description": None,
"completed_cycles": 2,
"error": None,
"duration": 0,
"remaining_asserts": ["A"],
"filename": "test.foo.yaml",
},
}
}
