def test_returning_multiple_matches_matching_multiple_dicts():
"""Test table() returns additional multiple matches as appended columns
The pattern ^extra, .* will match both "extra" and "extras"
"""
rows = [
{
"author": "Guybrush",
"notes": "Test row 1",
"extra": {
"first": "first extra",
"second": "second extra",
},
"extras": {
"third": "first extra",
"fourth": "second extra",
},
},
]
columns = collections.OrderedDict()
columns["Author"] = dict(pattern_path="^author$")
columns["Extras"] = dict(pattern_path=["^extra", ".*"],
return_multiple_columns=True)
csv_string = losser.table(rows, columns, csv=True)
nose.tools.assert_equals(csv_string,
'Author,extra_second,extra_first,extras_fourth,extras_third\r\n'
'Guybrush,second extra,first extra,second extra,first extra\r\n'
)
def test_table():
rows = [
dict(title="dataset one", extras=dict(update="hourly")),
dict(title="dataset two", extras=dict(Updated="daily")),
dict(title="dataset three", extras={"Update Frequency": "weekly"}),
]
columns = {
"Title": dict(pattern_path="^title$"),
"Update Frequency": dict(pattern_path=["^extras$", "update"]),
}
table = losser.table(rows, columns)
assert table == [
{
"Title": "dataset one",
"Update Frequency": "hourly"
},
{
"Title": "dataset two",
"Update Frequency": "daily"
},
{
"Title": "dataset three",
"Update Frequency": "weekly"
},
]
def test_returning_multiple_matches_matching_a_nested_dict():
"""Test that patterns matching a dictionary return a nested flattened dict
in this test the pattern is [extra, .*] which matches a dict
"""
return # TODO
rows = [
{
"author": "Guybrush Threepwood",
"notes": "Test row 1",
"extra": {
"first": {
"nested_extra": "nested first extra"
},
"second": {
"nested_extra": "nested second extra",
},
},
},
]
columns = collections.OrderedDict()
#columns["Author"] = dict(pattern_path="^author$")
columns["Extras"] = dict(pattern_path=["^extra", ".*"],
return_multiple_columns=True)
csv_string = losser.table(rows, columns, csv=True)
nose.tools.assert_equals(
csv_string, 'Author,second,first\r\n'
'Guybrush Threepwood,second extra,first extra\r\n'
'LeChuck,second extra,first extra\r\n'
'Herman Toothrot,second extra,first extra\r\n')
def test_returning_multiple_matches_matching_a_nested_dict():
"""Test that patterns matching a dictionary return a nested flattened dict
in this test the pattern is [extra, .*] which matches a dict
"""
return # TODO
rows = [
{
"author": "Guybrush Threepwood",
"notes": "Test row 1",
"extra": {
"first": {
"nested_extra": "nested first extra"
},
"second": {
"nested_extra": "nested second extra",
},
},
},
]
columns = collections.OrderedDict()
#columns["Author"] = dict(pattern_path="^author$")
columns["Extras"] = dict(pattern_path=["^extra", ".*"],
return_multiple_columns=True)
csv_string = losser.table(rows, columns, csv=True)
nose.tools.assert_equals(csv_string,
'Author,second,first\r\n'
'Guybrush Threepwood,second extra,first extra\r\n'
'LeChuck,second extra,first extra\r\n'
'Herman Toothrot,second extra,first extra\r\n'
)
def test_returning_multiple_matches_matching_multiple_dicts():
"""Test table() returns additional multiple matches as appended columns
The pattern ^extra, .* will match both "extra" and "extras"
"""
rows = [
{
"author": "Guybrush",
"notes": "Test row 1",
"extra": {
"first": "first extra",
"second": "second extra",
},
"extras": {
"third": "first extra",
"fourth": "second extra",
},
},
]
columns = collections.OrderedDict()
columns["Author"] = dict(pattern_path="^author$")
columns["Extras"] = dict(pattern_path=["^extra", ".*"],
return_multiple_columns=True)
csv_string = losser.table(rows, columns, csv=True)
nose.tools.assert_equals(
csv_string,
'Author,extra_second,extra_first,extras_fourth,extras_third\r\n'
'Guybrush,second extra,first extra,second extra,first extra\r\n')
def test_append_unused_keys():
"""Test the append_unused_keys option.
The values for any top-level keys in the input row that were not matched
by any of the patterns are appended to the end of the output row.
"""
return # TODO
rows = [
dict(
title="dataset one",
author="Mr. Author One",
description="A long description one",
extras=dict(update="hourly"),
),
dict(
title="dataset two",
author="Mr. Author One",
description="A long description one",
another_key="another value",
extras=dict(Updated="daily"),
),
dict(
title="dataset three",
author="Mr. Author One",
description="A long description one",
another_unused_key="An unused value",
extras={"Update Frequency": "weekly"},
),
]
columns = {
"Title": dict(pattern_path="^title$"),
"Update Frequency": dict(pattern_path=["^extras$", ".*"]),
}
table = losser.table(rows, columns, append_unused=True)
assert table == [
{
"Title": "dataset one",
"Update Frequency": "hourly",
"author": "Mr. Author One",
"description": "A long description one",
},
{
"Title": "dataset two",
"Update Frequency": "daily",
"author": "Mr. Author One",
"description": "A long description one",
"another_key": "another value",
},
{
"Title": "dataset three",
"Update Frequency": "weekly",
"author": "Mr. Author One",
"description": "A long description one",
"another_unused_key": "An unused value",
},
]
def test_max_length():
"""Test the max_length option."""
max_length = 5
rows = [dict(title="A really really long title")]
columns = {"Title": dict(pattern_path="^title$", max_length=max_length)}
table = losser.table(rows, columns)
assert len(table[0]["Title"]) <= max_length
def test_max_length():
"""Test the max_length option."""
max_length = 5
rows = [dict(title="A really really long title")]
columns = {"Title": dict(pattern_path="^title$", max_length=max_length)}
table = losser.table(rows, columns)
assert len(table[0]["Title"]) <= max_length
def test_append_unused_keys():
"""Test the append_unused_keys option.
The values for any top-level keys in the input row that were not matched
by any of the patterns are appended to the end of the output row.
"""
return # TODO
rows = [
dict(title="dataset one",
author="Mr. Author One",
description="A long description one",
extras=dict(update="hourly"),
),
dict(title="dataset two",
author="Mr. Author One",
description="A long description one",
another_key="another value",
extras=dict(Updated="daily"),
),
dict(title="dataset three",
author="Mr. Author One",
description="A long description one",
another_unused_key="An unused value",
extras={"Update Frequency": "weekly"},
),
]
columns = {
"Title": dict(pattern_path="^title$"),
"Update Frequency": dict(pattern_path=["^extras$", ".*"]),
}
table = losser.table(rows, columns, append_unused=True)
assert table == [
{"Title": "dataset one",
"Update Frequency": "hourly",
"author": "Mr. Author One",
"description": "A long description one",
},
{"Title": "dataset two",
"Update Frequency": "daily",
"author": "Mr. Author One",
"description": "A long description one",
"another_key": "another value",
},
{"Title": "dataset three",
"Update Frequency": "weekly",
"author": "Mr. Author One",
"description": "A long description one",
"another_unused_key": "An unused value",
},
]
def test_table_from_file():
"""Test table() when reading columns from file."""
rows = [
{
"author": "Guybrush Threepwood",
"notes": "Test row 1",
"resources": [
{"format": "CSV"},
{"format": "JSON"},
],
},
{
"author": "LeChuck",
"notes": "Test row 2",
"resources": [
{"format": "XLS"},
{"format": "XLS"},
],
},
{
"author": "Herman Toothrot",
"notes": "Test row 3",
"resources": [
{"format": "PDF"},
{"format": "TXT"},
],
},
]
path = os.path.join(_this_directory(), "test_columns.json")
table = losser.table(rows, path)
assert table == [
{
"Data Owner": "Guybrush Threepwood",
"Description": "Test row 1",
"Formats": ["CSV", "JSON"],
},
{
"Data Owner": "LeChuck",
"Description": "Test row 2",
"Formats": ["XLS"],
},
{
"Data Owner": "Herman Toothrot",
"Description": "Test row 3",
"Formats": ["PDF", "TXT"],
},
]
def test_returning_csv():
"""Test table() returning a CSV string."""
rows = [
{
"author": "Guybrush Threepwood",
"notes": "Test row 1",
"resources": [
{
"format": "CSV"
},
{
"format": "JSON"
},
],
},
{
"author": "LeChuck",
"notes": "Test row 2",
"resources": [
{
"format": "XLS"
},
{
"format": "XLS"
},
],
},
{
"author": "Herman Toothrot",
"notes": "Test row 3",
"resources": [
{
"format": "PDF"
},
{
"format": "TXT"
},
],
},
]
columns = collections.OrderedDict()
columns["Author"] = dict(pattern_path="^author$")
columns["Formats"] = dict(pattern_path=["^resources$", "^format$"])
csv_string = losser.table(rows, columns, csv=True)
assert csv_string == ("Author,Formats\r\n"
'Guybrush Threepwood,"CSV, JSON"\r\n'
'LeChuck,"XLS, XLS"\r\n'
'Herman Toothrot,"PDF, TXT"\r\n')
def test_blacklist():
"""Test the blacklist option.
When the append_unused_keys option is used, keys listed in the blacklist
will not be appended.
"""
return # TODO
rows = [
dict(
title="dataset one",
author="Mr. Author One",
description="A long description one",
extras=dict(update="hourly"),
secret="secret",
),
dict(
title="dataset two",
author="Mr. Author One",
description="A long description one",
another_key="another value",
extras=dict(Updated="daily"),
secret="secret",
),
dict(
title="dataset three",
author="Mr. Author One",
description="A long description one",
another_unused_key="An unused value",
extras={"Update Frequency": "weekly"},
top_secret="secret",
),
]
columns = {
"Title": dict(pattern_path="^title$"),
"Update Frequency": dict(pattern_path=["^extras$", "update"]),
}
blacklist = ["secret", "top_secret"]
table = losser.table(rows,
columns,
append_unused=True,
blacklist=blacklist)
for row in table:
for key in blacklist:
assert key not in row
def test_table():
rows = [
dict(title="dataset one", extras=dict(update="hourly")),
dict(title="dataset two", extras=dict(Updated="daily")),
dict(title="dataset three", extras={"Update Frequency": "weekly"}),
]
columns = {
"Title": dict(pattern_path="^title$"),
"Update Frequency": dict(pattern_path=["^extras$", "update"]),
}
table = losser.table(rows, columns)
assert table == [
{"Title": "dataset one", "Update Frequency": "hourly"},
{"Title": "dataset two", "Update Frequency": "daily"},
{"Title": "dataset three", "Update Frequency": "weekly"},
]
def test_blacklist():
"""Test the blacklist option.
When the append_unused_keys option is used, keys listed in the blacklist
will not be appended.
"""
return # TODO
rows = [
dict(title="dataset one",
author="Mr. Author One",
description="A long description one",
extras=dict(update="hourly"),
secret="secret",
),
dict(title="dataset two",
author="Mr. Author One",
description="A long description one",
another_key="another value",
extras=dict(Updated="daily"),
secret="secret",
),
dict(title="dataset three",
author="Mr. Author One",
description="A long description one",
another_unused_key="An unused value",
extras={"Update Frequency": "weekly"},
top_secret="secret",
),
]
columns = {
"Title": dict(pattern_path="^title$"),
"Update Frequency": dict(pattern_path=["^extras$", "update"]),
}
blacklist = ["secret", "top_secret"]
table = losser.table(rows, columns, append_unused=True,
blacklist=blacklist)
for row in table:
for key in blacklist:
assert key not in row
def test_returning_multiple_matches_as_extra_columns():
"""Test table() returns additional multiple matches as appended columns"""
rows = [
{
"author": "Guybrush Threepwood",
"notes": "Test row 1",
"extras": {
"first": "first extra",
"second": "second extra",
},
},
{
"author": "LeChuck",
"notes": "Test row 2",
"extras": {
"first": "first extra",
"second": "second extra",
},
},
{
"author": "Herman Toothrot",
"notes": "Test row 3",
"extras": {
"first": "first extra",
"second": "second extra",
},
},
]
columns = collections.OrderedDict()
columns["Author"] = dict(pattern_path="^author$")
columns["Extras"] = dict(pattern_path=["^extras$", ".*"],
return_multiple_columns=True)
csv_string = losser.table(rows, columns, csv=True)
nose.tools.assert_equals(csv_string,
'Author,extras_second,extras_first\r\n'
'Guybrush Threepwood,second extra,first extra\r\n'
'LeChuck,second extra,first extra\r\n'
'Herman Toothrot,second extra,first extra\r\n'
)
def test_returning_csv():
"""Test table() returning a CSV string."""
rows = [
{
"author": "Guybrush Threepwood",
"notes": "Test row 1",
"resources": [
{"format": "CSV"},
{"format": "JSON"},
],
},
{
"author": "LeChuck",
"notes": "Test row 2",
"resources": [
{"format": "XLS"},
{"format": "XLS"},
],
},
{
"author": "Herman Toothrot",
"notes": "Test row 3",
"resources": [
{"format": "PDF"},
{"format": "TXT"},
],
},
]
columns = collections.OrderedDict()
columns["Author"] = dict(pattern_path="^author$")
columns["Formats"] = dict(pattern_path=["^resources$", "^format$"])
csv_string = losser.table(rows, columns, csv=True)
assert csv_string == (
"Author,Formats\r\n"
'Guybrush Threepwood,"CSV, JSON"\r\n'
'LeChuck,"XLS, XLS"\r\n'
'Herman Toothrot,"PDF, TXT"\r\n'
)
def test_returning_multiple_matches_as_extra_columns():
"""Test table() returns additional multiple matches as appended columns"""
rows = [
{
"author": "Guybrush Threepwood",
"notes": "Test row 1",
"extras": {
"first": "first extra",
"second": "second extra",
},
},
{
"author": "LeChuck",
"notes": "Test row 2",
"extras": {
"first": "first extra",
"second": "second extra",
},
},
{
"author": "Herman Toothrot",
"notes": "Test row 3",
"extras": {
"first": "first extra",
"second": "second extra",
},
},
]
columns = collections.OrderedDict()
columns["Author"] = dict(pattern_path="^author$")
columns["Extras"] = dict(pattern_path=["^extras$", ".*"],
return_multiple_columns=True)
csv_string = losser.table(rows, columns, csv=True)
nose.tools.assert_equals(
csv_string, 'Author,extras_second,extras_first\r\n'
'Guybrush Threepwood,second extra,first extra\r\n'
'LeChuck,second extra,first extra\r\n'
'Herman Toothrot,second extra,first extra\r\n')
def test_table_from_file():
"""Test table() when reading columns from file."""
rows = [
{
"author": "Guybrush Threepwood",
"notes": "Test row 1",
"resources": [
{
"format": "CSV"
},
{
"format": "JSON"
},
],
},
{
"author": "LeChuck",
"notes": "Test row 2",
"resources": [
{
"format": "XLS"
},
{
"format": "XLS"
},
],
},
{
"author": "Herman Toothrot",
"notes": "Test row 3",
"resources": [
{
"format": "PDF"
},
{
"format": "TXT"
},
],
},
]
path = os.path.join(_this_directory(), "test_columns.json")
table = losser.table(rows, path)
assert table == [
{
"Data Owner": "Guybrush Threepwood",
"Description": "Test row 1",
"Formats": ["CSV", "JSON"],
},
{
"Data Owner": "LeChuck",
"Description": "Test row 2",
"Formats": ["XLS"],
},
{
"Data Owner": "Herman Toothrot",
"Description": "Test row 3",
"Formats": ["PDF", "TXT"],
},
]