def test_xml_check(self, env, result):
# Passing assertions
result.xml.check(element='<Root><Test>Foo</Test></Root>',
xpath='/Root/Test',
description='basic XML check')
result.xml.check(
element='<Root><Test>Value1</Test><Test>Value2</Test></Root>',
xpath='/Root/Test',
tags=['Value1', 'Value2'],
)
result.xml.check(
element='<Root><Test>Value1</Test><Test>Value2</Test></Root>',
xpath='/Root/Test',
tags=[cmp.In(['a', 'b', 'Value1']),
re.compile('.*lue2')],
)
result.xml.check(
element='''
<SOAP-ENV:Envelope
xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/">
<SOAP-ENV:Header/>
<SOAP-ENV:Body>
<ns0:message
xmlns:ns0="http://testplan">Hello world!</ns0:message>
</SOAP-ENV:Body>
</SOAP-ENV:Envelope>
''',
xpath='//*/a:message',
tags=['Hello*'],
namespaces={"a": "http://testplan"},
)
def test_dict_namespace(self, env, result):
# `result.dict` namespace can be used for applying advanced
# assertion rules to dictionaries, which can be nested.
actual = {
'foo': 1,
'bar': 2,
}
expected = {
'foo': 1,
'bar': 5,
'extra-key': 10,
}
# `dict.match` (recursively) matches elements of the dictionaries
result.dict.match(actual, expected, description='Simple dict match')
# `dict.match` supports nested data as well
actual = {'foo': {'alpha': [1, 2, 3], 'beta': {'color': 'red'}}}
expected = {'foo': {'alpha': [1, 2], 'beta': {'color': 'blue'}}}
result.dict.match(actual, expected, description='Nested dict match')
# It is possible to use custom comparators with `dict.match`
actual = {
'foo': [1, 2, 3],
'bar': {
'color': 'blue'
},
'baz': 'hello world',
}
expected = {
'foo': [1, 2, lambda v: isinstance(v, int)],
'bar': {
'color': comparison.In(['blue', 'red', 'yellow'])
},
'baz': re.compile(r'\w+ world'),
}
result.dict.match(actual,
expected,
description='Dict match: Custom comparators')
# `dict.check` can be used for checking existence / absence
# of keys within a dictionary
result.dict.check(dictionary={
'foo': 1,
'bar': 2,
'baz': 3,
},
has_keys=['foo', 'alpha'],
absent_keys=['bar', 'beta'])
def test_xml_check(self, env, result):
# Passing assertions
result.xml.check(
element="<Root><Test>Foo</Test></Root>",
xpath="/Root/Test",
description="basic XML check",
)
result.xml.check(
element="<Root><Test>Value1</Test><Test>Value2</Test></Root>",
xpath="/Root/Test",
tags=["Value1", "Value2"],
)
result.xml.check(
element="<Root><Test>Value1</Test><Test>Value2</Test></Root>",
xpath="/Root/Test",
tags=[cmp.In(["a", "b", "Value1"]),
re.compile(".*lue2")],
)
result.xml.check(
element="""
<SOAP-ENV:Envelope
xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/">
<SOAP-ENV:Header/>
<SOAP-ENV:Body>
<ns0:message
xmlns:ns0="http://testplan">Hello world!</ns0:message>
</SOAP-ENV:Body>
</SOAP-ENV:Envelope>
""",
xpath="//*/a:message",
tags=["Hello*"],
namespaces={"a": "http://testplan"},
)
def test_dict_namespace(self, env, result):
actual = {"foo": 1, "bar": 2}
expected = {"foo": 1, "bar": 5, "extra-key": 10}
# `dict.match` (recursively) matches elements of the dictionaries
result.dict.match(actual, expected, description="Simple dict match")
# `dict.match` supports nested data as well
actual = {"foo": {"alpha": [1, 2, 3], "beta": {"color": "red"}}}
expected = {"foo": {"alpha": [1, 2], "beta": {"color": "blue"}}}
result.dict.match(actual, expected, description="Nested dict match")
# It is possible to use custom comparators with `dict.match`
actual = {
"foo": [1, 2, 3],
"bar": {
"color": "blue"
},
"baz": "hello world",
}
expected = {
"foo": [1, 2, lambda v: isinstance(v, int)],
"bar": {
"color": comparison.In(["blue", "red", "yellow"])
},
"baz": re.compile(r"\w+ world"),
}
result.dict.match(actual,
expected,
description="Dict match: Custom comparators")
# You can also specify a comparator function to apply to all values in
# your dict. Standard comparators are available under
# testplan.common.utils.comparison.COMPARE_FUNCTIONS but any function
# f(x: Any, y: Any) -> bool can be used.
actual = {"foo": 1, "bar": 2, "baz": 3}
expected = {"foo": 1.0, "bar": 2.0, "baz": 3.0}
result.dict.match(
actual,
expected,
description="default assertion passes because the values are "
"numerically equal",
)
result.dict.match(
actual,
expected,
description="when we check types the assertion will fail",
value_cmp_func=comparison.COMPARE_FUNCTIONS["check_types"],
)
actual = {"foo": 1.02, "bar": 2.28, "baz": 3.50}
expected = {"foo": 0.98, "bar": 2.33, "baz": 3.46}
result.dict.match(
actual,
expected,
description="use a custom comparison function to check within a "
"tolerance",
value_cmp_func=lambda x, y: abs(x - y) < 0.1,
)
# The report_mode can be specified to limit the comparison
# information stored. By default all comparisons are stored and added
# to the report, but you can choose to discard some comparisons to
# reduce the size of the report when comparing very large dicts.
actual = {"key{}".format(i): i for i in range(10)}
expected = actual.copy()
expected["bad_key"] = "expected"
actual["bad_key"] = "actual"
result.dict.match(
actual,
expected,
description="only report the failing comparison",
report_mode=comparison.ReportOptions.FAILS_ONLY,
)
# `dict.check` can be used for checking existence / absence
# of keys within a dictionary
result.dict.check(
dictionary={
"foo": 1,
"bar": 2,
"baz": 3
},
has_keys=["foo", "alpha"],
absent_keys=["bar", "beta"],
)
# `dict.log` can be used to log a dictionary in human readable format.
result.dict.log(dictionary={
"foo": [1, 2, 3],
"bar": {
"color": "blue"
},
"baz": "hello world",
})
def test_table_namespace(self, env, result):
list_of_dicts = [
{
"name": "Bob",
"age": 32
},
{
"name": "Susan",
"age": 24
},
{
"name": "Rick",
"age": 67
},
]
list_of_lists = [
["name", "age"],
["Bob", 32],
["Susan", 24],
["Rick", 67],
]
sample_table = [
["symbol", "amount"],
["AAPL", 12],
["GOOG", 21],
["FB", 32],
["AMZN", 5],
["MSFT", 42],
]
large_table = [sample_table[0]] + sample_table[1:] * 100
# We can log the table using result.table.log, either a list of dicts
# or a list of lists
result.table.log(list_of_dicts, description="Table Log: list of dicts")
result.table.log(
list_of_lists,
display_index=True,
description="Table Log: list of lists",
)
result.table.log(list_of_lists[:1], description="Empty table")
result.table.log(
[{
"name": "Bob",
"age": 32
}, {
"name": "Susan"
}],
description="Empty cell",
)
result.table.log([[1, 2, 3], ["abc", "def", "xyz"]],
description="Non-string header")
# When tables with over 10 rows are logged:
# * In the PDF report, only the first and last 5 rows are shown. The
# row indices are then also shown by default.
# * In console out the entire table will be shown, without indices.
result.table.log(large_table[:21], description="Table Log: many rows")
# When tables are too wide:
# * In the PDF report, the columns are split into tables over multiple
# rows. The row indices are then also shown by default.
# * In console out the table will be shown as is, if the formatting
# looks odd the output can be piped into a file.
columns = [["col_{}".format(i) for i in range(20)]]
rows = [["row {} col {}".format(i, j) for j in range(20)]
for i in range(10)]
result.table.log(columns + rows, description="Table Log: many columns")
# When the cell values exceed the character limit:
# * In the PDF report they will be truncated and appended with '...'.
# * In console out, should they also be truncated?
long_cell_table = [
["Name", "Age", "Address"],
["Bob Stevens", "33", "89 Trinsdale Avenue, LONDON, E8 0XW"],
["Susan Evans", "21", "100 Loop Road, SWANSEA, U8 12JK"],
["Trevor Dune", "88", "28 Kings Lane, MANCHESTER, MT16 2YT"],
["Belinda Baggins", "38", "31 Prospect Hill, DOYNTON, BS30 9DN"],
["Cosimo Hornblower", "89", "65 Prospect Hill, SURREY, PH33 4TY"],
["Sabine Wurfel", "31", "88 Clasper Way, HEXWORTHY, PL20 4BG"],
]
result.table.log(long_cell_table, description="Table Log: long cells")
# Add external/internal link in the table log
result.table.log(
[
["Description", "Data"],
[
"External Link",
LogLink(link="https://www.google.com", title="Google"),
],
# Require plan.runnable.disable_reset_report_uid() in main function
# to avoid generating uuid4 as the report uid so that we can use
# the test name as the link in the report.
[
"Internal Link",
LogLink(
link=
"/Assertions%20Test/SampleSuite/test_basic_assertions",
title="test_basic_assertions",
inner=True,
),
],
],
description="Link to external/internal",
)
# Customize formatted value in the table log
result.table.log(
[
["Description", "Data"],
[
"Formatted Value - 0.6",
FormattedValue(display="60%", value=0.6),
],
[
"Formatted Value - 0.08",
FormattedValue(display="8%", value=0.08),
],
],
description="Formatted value",
)
result.table.match(
list_of_lists,
list_of_lists,
description="Table Match: list of list vs list of list",
)
result.table.match(
list_of_dicts,
list_of_dicts,
description="Table Match: list of dict vs list of dict",
)
result.table.match(
list_of_dicts,
list_of_lists,
description="Table Match: list of dict vs list of list",
)
result.table.diff(
list_of_lists,
list_of_lists,
description="Table Diff: list of list vs list of list",
)
result.table.diff(
list_of_dicts,
list_of_dicts,
description="Table Diff: list of dict vs list of dict",
)
result.table.diff(
list_of_dicts,
list_of_lists,
description="Table Diff: list of dict vs list of list",
)
# For table match, Testplan allows use of custom comparators
# (callables & regex) instead of plain value matching
actual_table = [
["name", "age"],
["Bob", 32],
["Susan", 24],
["Rick", 67],
]
expected_table = [
["name", "age"],
# Regex match for row 1, name column
# Callable match for row 1, age column
[re.compile(r"\w{3}"), lambda age: 30 < age < 40],
["Susan", 24], # simple match with exact values for row 2
# Callable match for row 3 name column
# Simple match for row 3 age column
[lambda name: name in ["David", "Helen", "Pablo"], 67],
]
result.table.match(
actual_table,
expected_table,
description="Table Match: simple comparators",
)
result.table.diff(
actual_table,
expected_table,
description="Table Diff: simple comparators",
)
# Equivalent assertion as above, using Testplan's custom comparators
# These utilities produce more readable output
expected_table_2 = [
["name", "age"],
[
re.compile(r"\w{3}"),
comparison.Greater(30) & comparison.Less(40),
],
["Susan", 24],
[comparison.In(["David", "Helen", "Pablo"]), 67],
]
result.table.match(
actual_table,
expected_table_2,
description="Table Match: readable comparators",
)
result.table.diff(
actual_table,
expected_table_2,
description="Table Diff: readable comparators",
)
# While comparing tables with large number of columns
# we can 'trim' some of the columns to get more readable output
table_with_many_columns = [{
"column_{}".format(idx): i * idx
for idx in range(30)
} for i in range(10)]
# Only use 2 columns for comparison, trim the rest
result.table.match(
table_with_many_columns,
table_with_many_columns,
include_columns=["column_1", "column_2"],
report_all=False,
description="Table Match: Trimmed columns",
)
result.table.diff(
table_with_many_columns,
table_with_many_columns,
include_columns=["column_1", "column_2"],
report_all=False,
description="Table Diff: Trimmed columns",
)
# While comparing tables with large number of rows
# we can stop comparing if the number of failed rows exceeds the limit
matching_rows_1 = [{
"amount": idx * 10,
"product_id": random.randint(1000, 5000)
} for idx in range(5)]
matching_rows_2 = [{
"amount": idx * 10,
"product_id": random.randint(1000, 5000)
} for idx in range(500)]
row_diff_a = [
{
"amount": 25,
"product_id": 1111
},
{
"amount": 20,
"product_id": 2222
},
{
"amount": 50,
"product_id": 3333
},
]
row_diff_b = [
{
"amount": 35,
"product_id": 1111
},
{
"amount": 20,
"product_id": 1234
},
{
"amount": 20,
"product_id": 5432
},
]
table_a = matching_rows_1 + row_diff_a + matching_rows_2
table_b = matching_rows_1 + row_diff_b + matching_rows_2
# We can 'trim' some rows and display at most 2 rows of failures
result.table.match(
table_a,
table_b,
fail_limit=2,
report_all=False,
description="Table Match: Trimmed rows",
)
# Only display mismatching rows, with a maximum limit of 2 rows
result.table.diff(
table_a,
table_b,
fail_limit=2,
report_all=False,
description="Table Diff: Trimmed rows",
)
# result.table.column_contain can be used for checking if all of the
# cells on a table's column exists in a given list of values
result.table.column_contain(values=["AAPL", "AMZN"],
table=sample_table,
column="symbol")
# We can use `limit` and `report_fails_only` arguments for producing
# less output for large tables
result.table.column_contain(
values=["AAPL", "AMZN"],
table=large_table,
column="symbol",
limit=20, # Process 50 items at most
report_fails_only=True, # Only include failures in the result
)
def test_xml_check(self, env, result):
# Passing assertions
result.xml.check(element='<Root><Test>Foo</Test></Root>',
xpath='/Root/Test',
description='basic XML check')
result.xml.check(
element='<Root><Test>Value1</Test><Test>Value2</Test></Root>',
xpath='/Root/Test',
tags=['Value1', 'Value2'],
)
result.xml.check(
element='<Root><Test>Value1</Test><Test>Value2</Test></Root>',
xpath='/Root/Test',
tags=[cmp.In(['a', 'b', 'Value1']),
re.compile('.*lue2')],
)
result.xml.check(
element='''
<SOAP-ENV:Envelope
xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/">
<SOAP-ENV:Header/>
<SOAP-ENV:Body>
<ns0:message
xmlns:ns0="http://testplan">Hello world!</ns0:message>
</SOAP-ENV:Body>
</SOAP-ENV:Envelope>
''',
xpath='//*/a:message',
tags=['Hello*'],
namespaces={"a": "http://testplan"},
)
# Failing assertions
result.xml.check(
element='<Root><Test>Foo</Test></Root>',
xpath='/Root/Bar',
)
result.xml.check(
element='''
<Root>
<Test>Foo</Test>
<Test>Bar</Test>
<Test>Baz</Test>
</Root>
''',
xpath='/Root/Test',
tags=['Foo', 'Alpha', 'Beta'],
)
result.xml.check(
element='''
<Root>
<Test></Test>
</Root>
''',
xpath='/Root/Test',
tags=['Foo'],
)
result.xml.check(
element='''
<Root>
<Test>Foo</Test>
</Root>
''',
xpath='/Root/Test',
tags=['Foo', 'Bar'],
)
result.xml.check(
element='<Root><Test>Value1</Test><Test>Value2</Test></Root>',
xpath='/Root/Test',
tags=[cmp.In(['a', 'b', 'c']),
re.compile('Foo')],
)
result.xml.check(element='''
<SOAP-ENV:Envelope
xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/">
<SOAP-ENV:Header/>
<SOAP-ENV:Body>
<ns0:message
xmlns:ns0="mismatch-ns">Hello world!</ns0:message>
<ns0:message
xmlns:ns0="http://testplan">Foobar</ns0:message>
</SOAP-ENV:Body>
</SOAP-ENV:Envelope>
''',
xpath='//*/a:message',
tags=['Hello*'],
namespaces={"a": "http://testplan"})
# Error raised during match via custom callable
result.xml.check(
element='''
<Root>
<Test>Foo</Test>
</Root>
''',
xpath='/Root/Test',
tags=[error_func],
)
def test_fix_namespace(self, env, result):
# `fix.match` can compare two fix messages, and
# supports custom comparators (like `dict.match`)
fix_msg_1 = {
36: 6,
22: 5,
55: 2,
38: 5,
555: [
{
600: "A",
601: "A",
683: [{688: "a", 689: None}, {688: "b", 689: "b"}],
},
{
600: "B",
601: "B",
683: [{688: "c", 689: "c"}, {688: "d", 689: "d"}],
},
],
}
fix_msg_2 = {
36: 6,
22: 5,
55: 2,
38: comparison.GreaterEqual(4),
555: [
{
600: "A",
601: "B",
683: [
{688: "a", 689: re.compile(r"[a-z]")},
{688: "b", 689: "b"},
],
},
{
600: "C",
601: "B",
683: [
{688: "c", 689: comparison.In(("c", "d"))},
{688: "d", 689: "d"},
],
},
],
}
result.fix.match(fix_msg_1, fix_msg_2)
# `fix.check` can be used for checking existence / absence
# of certain tags in a fix message
result.fix.check(
msg=fix_msg_1, has_tags=[26, 22, 11], absent_tags=[444, 555]
)
# `fix.log` can be used to log a fix message in human readable format.
result.fix.log(
msg={
36: 6,
22: 5,
55: 2,
38: 5,
555: [{556: "USD", 624: 1}, {556: "EUR", 624: 2}],
}
)
def test_fix_namespace(self, env, result):
# `result.fix` namespace can be used for applying advanced
# assertion rules to fix messages, which can
# be nested (e.g. repeating groups)
# For more info about FIX protocol, please see:
# https://en.wikipedia.org/wiki/Financial_Information_eXchange
# `fix.match` can compare two fix messages, and
# supports custom comparators (like `dict.match`)
fix_msg_1 = {
36:
6,
22:
5,
55:
2,
38:
5,
555: [
{
600: "A",
601: "A",
683: [{
688: "a",
689: "a"
}, {
688: "b",
689: "b"
}],
},
{
600: "B",
601: "B",
683: [{
688: "c",
689: "c"
}, {
688: "d",
689: "d"
}],
},
],
}
fix_msg_2 = {
36:
6,
22:
5,
55:
2,
38:
comparison.GreaterEqual(4),
555: [
{
600:
"A",
601:
"B",
683: [
{
688: "a",
689: re.compile(r"[a-z]")
},
{
688: "b",
689: "b"
},
],
},
{
600:
"C",
601:
"B",
683: [
{
688: "c",
689: comparison.In(("c", "d"))
},
{
688: "d",
689: "d"
},
],
},
],
}
result.fix.match(fix_msg_1, fix_msg_2)
# `fix.check` can be used for checking existence / absence
# of certain tags in a fix message
result.fix.check(msg=fix_msg_1,
has_tags=[26, 22, 11],
absent_tags=[444, 555])
# `fix.log` can be used to log a fix message in human readable format.
result.fix.log(
msg={
36: 6,
22: 5,
55: 2,
38: 5,
555: [{
556: "USD",
624: 1
}, {
556: "EUR",
624: 2
}],
})
def test_fix_namespace(self, env, result):
# `result.fix` namespace can be used for applying advanced
# assertion rules to fix messages, which can
# be nested (e.g. repeating groups)
# For more info about FIX protocol, please see:
# https://en.wikipedia.org/wiki/Financial_Information_eXchange
# `fix.match` can compare two fix messages, and
# supports custom comparators (like `dict.match`)
fix_msg_1 = {
36: 6,
22: 5,
55: 2,
38: 5,
555: [
{
600: 'A',
601: 'A',
683: [
{
688: 'a',
689: 'a'
},
{
688: 'b',
689: 'b'
}
]
},
{
600: 'B',
601: 'B',
683: [
{
688: 'c',
689: 'c'
},
{
688: 'd',
689: 'd'
}
]
}
]
}
fix_msg_2 = {
36: 6,
22: 5,
55: 2,
38: comparison.GreaterEqual(4),
555: [
{
600: 'A',
601: 'B',
683: [
{
688: 'a',
689: re.compile(r'[a-z]')
},
{
688: 'b',
689: 'b'
}
]
},
{
600: 'C',
601: 'B',
683: [
{
688: 'c',
689: comparison.In(('c', 'd'))
},
{
688: 'd',
689: 'd'
}
]
}
]
}
result.fix.match(fix_msg_1, fix_msg_2)
# `fix.check` can be used for checking existence / absence
# of certain tags in a fix message
result.fix.check(
msg=fix_msg_1,
has_tags=[26, 22, 11],
absent_tags=[444, 555],
)
def test_table_namespace(self, env, result):
# We can use `result.table` namespace to apply table specific checks.
# 1- A table is represented either as a
# list of dictionaries with uniform keys (columns)
# 2- Or a list of lists that have columns as the first item and the
# row values as the rest
list_of_dicts = [
{'name': 'Bob', 'age': 32},
{'name': 'Susan', 'age': 24},
{'name': 'Rick', 'age': 67},
]
list_of_lists = [
['name', 'age'],
['Bob', 32],
['Susan', 24],
['Rick', 67]
]
result.table.match(
list_of_lists, list_of_lists,
description='Table Match: list of list vs list of list'
)
result.table.match(
list_of_dicts, list_of_dicts,
description='Table Match: list of dict vs list of dict'
)
result.table.match(
list_of_dicts, list_of_lists,
description='Table Match: list of dict vs list of list'
)
# For table match, Testplan allows use of custom comparators
# (callables & regex) instead of plain value matching
actual_table = [
['name', 'age'],
['Bob', 32],
['Susan', 24],
['Rick', 67]
]
expected_table = [
['name', 'age'],
# Regex match for row 1, name column
# Callable match for row 1, age column
[re.compile(r'\w{3}'), lambda age: 30 < age < 40],
['Susan', 24], # simple match with exact values for row 2
# Callable match for row 3 name column
# Simple match for row 3 age column
[
lambda name: name in ['David', 'Helen', 'Pablo'],
67,
]
]
result.table.match(
actual_table, expected_table,
description='Table Match: simple comparators'
)
# Equivalent assertion as above, using Testplan's custom comparators
# These utilities produce more readable output
expected_table_2 = [
['name', 'age'],
[
re.compile(r'\w{3}'),
comparison.Greater(30) & comparison.Less(40)
],
['Susan', 24],
[comparison.In(['David', 'Helen', 'Pablo']), 67]
]
result.table.match(
actual_table, expected_table_2,
description='Table Match: readable comparators'
)
# While comparing tables with large number of columns
# we can 'trim' some of the columns to get more readable output
table_with_many_columns = [
{'column_{}'.format(idx): i * idx for idx in range(30)}
for i in range(10)
]
# Only use 2 columns for comparison, trim the rest
result.table.match(
table_with_many_columns,
table_with_many_columns,
include_columns=['column_1', 'column_2'],
report_all=False,
description='Table Match: Trimmed columns'
)
# While comparing tables with large number of rows
# we can 'trim' some rows and display a limited number of failures only
matching_rows = [
{'amount': idx * 10, 'product_id': random.randint(1000, 5000)}
for idx in range(500)
]
row_diff_a = [
{'amount': 25, 'product_id': 1111},
{'amount': 20, 'product_id': 2222},
{'amount': 50, 'product_id': 3333},
]
row_diff_b = [
{'amount': 35, 'product_id': 1111},
{'amount': 20, 'product_id': 1234},
{'amount': 20, 'product_id': 5432},
]
table_a = matching_rows + row_diff_a + matching_rows
table_b = matching_rows + row_diff_b + matching_rows
# Only display mismatching rows, with a maximum limit of 2 rows
result.table.match(
table_a,
table_b,
fail_limit=2,
report_all=False,
description='Table Match: Trimmed rows'
)
# result.table.column_contain can be used for checking if all of the
# cells on a table's column exists in a given list of values
sample_table = [
['symbol', 'amount'],
['AAPL', 12],
['GOOG', 21],
['FB', 32],
['AMZN', 5],
['MSFT', 42]
]
result.table.column_contain(
values=['AAPL', 'AMZN'],
table=sample_table,
column='symbol',
)
# We can use `limit` and `report_fails_only` arguments for producing
# less output for large tables
large_table = [sample_table[0]] + sample_table[1:] * 100
result.table.column_contain(
values=['AAPL', 'AMZN'],
table=large_table,
column='symbol',
limit=20, # Process 50 items at most
report_fails_only=True, # Only include failures in the result
)
# We can log the table using result.table.log, either a list of dicts
# or a list of lists
result.table.log(list_of_dicts, description='Table Log: list of dicts')
result.table.log(list_of_lists, description='Table Log: list of lists')
# When tables with over 10 rows are logged:
# * In the PDF report, only the first and last 5 rows are shown. The
# row indices are then also shown by default.
# * In console out the entire table will be shown, without indices.
result.table.log(large_table[:21], description='Table Log: many rows')
# When tables are too wide:
# * In the PDF report, the columns are split into tables over multiple
# rows. The row indices are then also shown by default.
# * In console out the table will be shown as is, if the formatting
# looks odd the output can be piped into a file.
columns = [['col_{}'.format(i) for i in range(20)]]
rows = [['row {} col {}'.format(i, j)
for j in range(20)]
for i in range(10)]
result.table.log(columns + rows, description='Table Log: many columns')
# When the cell values exceed the character limit:
# * In the PDF report they will be truncated and appended with '...'.
# * In console out, should they also be truncated?
long_cell_table = [
['Name', 'Age', 'Address'],
['Bob Stevens', '33', '89 Trinsdale Avenue, LONDON, E8 0XW'],
['Susan Evans', '21', '100 Loop Road, SWANSEA, U8 12JK'],
['Trevor Dune', '88', '28 Kings Lane, MANCHESTER, MT16 2YT'],
['Belinda Baggins', '38', '31 Prospect Hill, DOYNTON, BS30 9DN'],
['Cosimo Hornblower', '89', '65 Prospect Hill, SURREY, PH33 4TY'],
['Sabine Wurfel', '31', '88 Clasper Way, HEXWORTHY, PL20 4BG'],
]
result.table.log(long_cell_table, description='Table Log: long cells')
def test_xml_check(self, env, result):
# Passing assertions
result.xml.check(
element="<Root><Test>Foo</Test></Root>",
xpath="/Root/Test",
description="basic XML check",
)
result.xml.check(
element="<Root><Test>Value1</Test><Test>Value2</Test></Root>",
xpath="/Root/Test",
tags=["Value1", "Value2"],
)
result.xml.check(
element="<Root><Test>Value1</Test><Test>Value2</Test></Root>",
xpath="/Root/Test",
tags=[cmp.In(["a", "b", "Value1"]),
re.compile(".*lue2")],
)
result.xml.check(
element="""
<SOAP-ENV:Envelope
xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/">
<SOAP-ENV:Header/>
<SOAP-ENV:Body>
<ns0:message
xmlns:ns0="http://testplan">Hello world!</ns0:message>
</SOAP-ENV:Body>
</SOAP-ENV:Envelope>
""",
xpath="//*/a:message",
tags=["Hello*"],
namespaces={"a": "http://testplan"},
)
# Failing assertions
result.xml.check(element="<Root><Test>Foo</Test></Root>",
xpath="/Root/Bar")
result.xml.check(
element="""
<Root>
<Test>Foo</Test>
<Test>Bar</Test>
<Test>Baz</Test>
</Root>
""",
xpath="/Root/Test",
tags=["Foo", "Alpha", "Beta"],
)
result.xml.check(
element="""
<Root>
<Test></Test>
</Root>
""",
xpath="/Root/Test",
tags=["Foo"],
)
result.xml.check(
element="""
<Root>
<Test>Foo</Test>
</Root>
""",
xpath="/Root/Test",
tags=["Foo", "Bar"],
)
result.xml.check(
element="<Root><Test>Value1</Test><Test>Value2</Test></Root>",
xpath="/Root/Test",
tags=[cmp.In(["a", "b", "c"]),
re.compile("Foo")],
)
result.xml.check(
element="""
<SOAP-ENV:Envelope
xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/">
<SOAP-ENV:Header/>
<SOAP-ENV:Body>
<ns0:message
xmlns:ns0="mismatch-ns">Hello world!</ns0:message>
<ns0:message
xmlns:ns0="http://testplan">Foobar</ns0:message>
</SOAP-ENV:Body>
</SOAP-ENV:Envelope>
""",
xpath="//*/a:message",
tags=["Hello*"],
namespaces={"a": "http://testplan"},
)
# Error raised during match via custom callable
result.xml.check(
element="""
<Root>
<Test>Foo</Test>
</Root>
""",
xpath="/Root/Test",
tags=[error_func],
)
def test_custom_callable():
custom_callable = cmp.Custom(lambda value: value % 2 == 0,
description="Value is even.")
assert custom_callable(4) == True
assert str(custom_callable) == "Value is even."
@pytest.mark.parametrize(
"composed_callable,value,expected,description",
(
(cmp.LessEqual(5) & cmp.Greater(2), 4, True, "(VAL <= 5 and VAL > 2)"),
(
cmp.In([1, 2, 3]) | cmp.Equal(None),
None,
True,
"(VAL in [1, 2, 3] or VAL == None)",
),
(
cmp.And(
cmp.Or(cmp.Equal("foo"), cmp.In([1, 2, 3]), cmp.Less(10)),
cmp.Or(cmp.Greater(5), cmp.IsFalse()),
),
8,
True,
"((VAL == foo or VAL in [1, 2, 3] or "
"VAL < 10) and (VAL > 5 or bool(VAL) is False))",
),
),
def test_dict_namespace(self, env, result):
# `result.dict` namespace can be used for applying advanced
# assertion rules to dictionaries, which can be nested.
actual = {
'foo': 1,
'bar': 2,
}
expected = {
'foo': 1,
'bar': 5,
'extra-key': 10,
}
# `dict.match` (recursively) matches elements of the dictionaries
result.dict.match(actual, expected, description='Simple dict match')
# `dict.match` supports nested data as well
actual = {'foo': {'alpha': [1, 2, 3], 'beta': {'color': 'red'}}}
expected = {'foo': {'alpha': [1, 2], 'beta': {'color': 'blue'}}}
result.dict.match(actual, expected, description='Nested dict match')
# It is possible to use custom comparators with `dict.match`
actual = {
'foo': [1, 2, 3],
'bar': {
'color': 'blue'
},
'baz': 'hello world',
}
expected = {
'foo': [1, 2, lambda v: isinstance(v, int)],
'bar': {
'color': comparison.In(['blue', 'red', 'yellow'])
},
'baz': re.compile(r'\w+ world'),
}
result.dict.match(actual,
expected,
description='Dict match: Custom comparators')
# You can also specify a comparator function to apply to all values in
# your dict. Standard comparators are available under
# testplan.common.utils.comparison.COMPARE_FUNCTIONS but any function
# f(x: Any, y: Any) -> bool can be used.
actual = {'foo': 1, 'bar': 2, 'baz': 3}
expected = {'foo': 1.0, 'bar': 2.0, 'baz': 3.0}
result.dict.match(
actual,
expected,
description='default assertion passes because the values are '
'numerically equal')
result.dict.match(
actual,
expected,
description='when we check types the assertion will fail',
value_cmp_func=comparison.COMPARE_FUNCTIONS['check_types'])
actual = {'foo': 1.02, 'bar': 2.28, 'baz': 3.50}
expected = {'foo': 0.98, 'bar': 2.33, 'baz': 3.46}
result.dict.match(
actual,
expected,
description='use a custom comparison function to check within a '
'tolerance',
value_cmp_func=lambda x, y: abs(x - y) < 0.1)
# The report_mode can be specified to limit the comparison
# information stored. By default all comparisons are stored and added
# to the report, but you can choose to discard some comparisons to
# reduce the size of the report when comparing very large dicts.
actual = {'key{}'.format(i): i for i in range(10)}
expected = actual.copy()
expected['bad_key'] = 'expected'
actual['bad_key'] = 'actual'
result.dict.match(actual,
expected,
description='only report the failing comparison',
report_mode=comparison.ReportOptions.FAILS_ONLY)
# `dict.check` can be used for checking existence / absence
# of keys within a dictionary
result.dict.check(dictionary={
'foo': 1,
'bar': 2,
'baz': 3,
},
has_keys=['foo', 'alpha'],
absent_keys=['bar', 'beta'])
# `dict.log` can be used to log a dictionary in human readable format.
result.dict.log(dictionary={
'foo': [1, 2, 3],
'bar': {
'color': 'blue'
},
'baz': 'hello world',
})
assert callable_obj(value) == expected
assert str(callable_obj) == description
def test_custom_callable():
custom_callable = cmp.Custom(lambda value: value % 2 == 0,
description='Value is even.')
assert custom_callable(4) == True
assert str(custom_callable) == 'Value is even.'
@pytest.mark.parametrize(
'composed_callable,value,expected,description',
((cmp.LessEqual(5) & cmp.Greater(2), 4, True, '(VAL <= 5 and VAL > 2)'),
(cmp.In([1, 2, 3]) | cmp.Equal(None), None, True,
'(VAL in [1, 2, 3] or VAL == None)'), (
cmp.And(cmp.Or(
cmp.Equal('foo'),
cmp.In([1, 2, 3]),
cmp.Less(10),
), cmp.Or(cmp.Greater(5), cmp.IsFalse())),
8,
True,
"((VAL == foo or VAL in [1, 2, 3] or "
"VAL < 10) and (VAL > 5 or bool(VAL) is False))",
)))
def test_comparator_composition(composed_callable, value, expected,
description):
assert composed_callable(value) == expected
assert str(composed_callable) == description
