def test_flatten():
data = {
"extras": [
{"key": "genre", "value": u"horror"},
{"key": "media", "value": u"dvd"},
],
"license_id": u"gpl-3.0",
"name": u"testpkg",
"resources": [
{
u"alt_url": u"alt_url",
u"description": u"Second file",
u"extras": {u"size": u"200"},
u"format": u"xml",
u"hash": u"def123",
u"url": u"http://blah.com/file2.xml",
},
{
u"alt_url": u"alt_url",
u"description": u"Main file",
u"extras": {u"size": u"200"},
u"format": u"xml",
u"hash": u"abc123",
u"url": u"http://blah.com/file.xml",
},
],
"tags": [{"name": u"russion"}, {"name": u"novel"}],
"title": u"Some Title",
"url": u"http://blahblahblah.mydomain",
}
assert flatten_dict(data) == {
("extras", 0, "key"): "genre",
("extras", 0, "value"): u"horror",
("extras", 1, "key"): "media",
("extras", 1, "value"): u"dvd",
("license_id",): u"gpl-3.0",
("name",): u"testpkg",
("resources", 0, u"alt_url"): u"alt_url",
("resources", 0, u"description"): u"Second file",
("resources", 0, u"extras"): {u"size": u"200"},
("resources", 0, u"format"): u"xml",
("resources", 0, u"hash"): u"def123",
("resources", 0, u"url"): u"http://blah.com/file2.xml",
("resources", 1, u"alt_url"): u"alt_url",
("resources", 1, u"description"): u"Main file",
("resources", 1, u"extras"): {u"size": u"200"},
("resources", 1, u"format"): u"xml",
("resources", 1, u"hash"): u"abc123",
("resources", 1, u"url"): u"http://blah.com/file.xml",
("tags", 0, "name"): u"russion",
("tags", 1, "name"): u"novel",
("title",): u"Some Title",
("url",): u"http://blahblahblah.mydomain",
}, pformat(flatten_dict(data))
assert data == unflatten(flatten_dict(data))
def test_flatten(self):
data = {
"extras": [{"key": "genre", "value": u"horror"}, {"key": "media", "value": u"dvd"}],
"license_id": u"gpl-3.0",
"name": u"testpkg",
"resources": [
{
u"alt_url": u"alt_url",
u"description": u"Second file",
u"extras": {u"size": u"200"},
u"format": u"xml",
u"hash": u"def123",
u"url": u"http://blah.com/file2.xml",
},
{
u"alt_url": u"alt_url",
u"description": u"Main file",
u"extras": {u"size": u"200"},
u"format": u"xml",
u"hash": u"abc123",
u"url": u"http://blah.com/file.xml",
},
],
"tags": [{"name": u"russion"}, {"name": u"novel"}],
"title": u"Some Title",
"url": u"http://blahblahblah.mydomain",
}
assert flatten_dict(data) == {
("extras", 0, "key"): "genre",
("extras", 0, "value"): u"horror",
("extras", 1, "key"): "media",
("extras", 1, "value"): u"dvd",
("license_id",): u"gpl-3.0",
("name",): u"testpkg",
("resources", 0, u"alt_url"): u"alt_url",
("resources", 0, u"description"): u"Second file",
("resources", 0, u"extras"): {u"size": u"200"},
("resources", 0, u"format"): u"xml",
("resources", 0, u"hash"): u"def123",
("resources", 0, u"url"): u"http://blah.com/file2.xml",
("resources", 1, u"alt_url"): u"alt_url",
("resources", 1, u"description"): u"Main file",
("resources", 1, u"extras"): {u"size": u"200"},
("resources", 1, u"format"): u"xml",
("resources", 1, u"hash"): u"abc123",
("resources", 1, u"url"): u"http://blah.com/file.xml",
("tags", 0, "name"): u"russion",
("tags", 1, "name"): u"novel",
("title",): u"Some Title",
("url",): u"http://blahblahblah.mydomain",
}, pformat(flatten_dict(data))
assert data == unflatten(flatten_dict(data))
def test_if_empty_guess_format(self):
data = {'name': 'package_name', 'resources': [
{'url': 'http://fakedomain/my.csv', 'format': ''},
{'url': 'http://fakedomain/my.pdf',
'format': df.Missing},
{'url': 'http://fakedomain/my.pdf', 'format': 'pdf'},
{'url': 'http://fakedomain/my.pdf',
'id': 'fake_resource_id', 'format': ''}
]}
data = df.flatten_dict(data)
@t.does_not_modify_errors_dict
def call_validator(*args, **kwargs):
return validators.if_empty_guess_format(*args, **kwargs)
new_data = copy.deepcopy(data)
call_validator(key=('resources', 0, 'format'), data=new_data,
errors={}, context={})
assert new_data[('resources', 0, 'format')] == 'text/csv'
new_data = copy.deepcopy(data)
call_validator(key=('resources', 1, 'format'), data=new_data,
errors={}, context={})
assert new_data[('resources', 1, 'format')] == 'application/pdf'
new_data = copy.deepcopy(data)
call_validator(key=('resources', 2, 'format'), data=new_data,
errors={}, context={})
assert new_data[('resources', 2, 'format')] == 'pdf'
new_data = copy.deepcopy(data)
call_validator(key=('resources', 3, 'format'), data=new_data,
errors={}, context={})
assert new_data[('resources', 3, 'format')] == ''
def test_with_two_resource_types_defined(self):
"""
Tests with two resource types defined.
"""
data = {
"additional_resources": [
{"description": "Additional 1", "url": "A_Url 1"},
{"description": "Additional 2", "url": "A_Url 2"},
],
"timeseries_resources": [
{"description": "Timeseries 1", "url": "T_Url 1", "date": "T_Date 1"},
{"description": "Timeseries 2", "url": "T_Url 2", "date": "T_Date 2"},
{"description": "Timeseries 3", "url": "T_Url 3", "date": "T_Date 3"},
],
}
flattened_data = flatten_dict(data)
ignored = {}
errors = dict([(k, []) for k in flattened_data.keys()])
merge_resources(("__after",), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
"resources": [
{"description": "Additional 1", "url": "A_Url 1"},
{"description": "Additional 2", "url": "A_Url 2"},
{"description": "Timeseries 1", "url": "T_Url 1", "date": "T_Date 1"},
{"description": "Timeseries 2", "url": "T_Url 2", "date": "T_Date 2"},
{"description": "Timeseries 3", "url": "T_Url 3", "date": "T_Date 3"},
]
}
assert_equal(result_data, expected_data)
def test_remove_blank_resources(self):
"""
Blank rows occur as a result of switching to/from timeseries resource
or deleting a resource in a table (you can't remove the row itself).
Therefore remove blank rows.
"""
import nose
nose.tools.maxDiff = 2000
data = {
"additional_resources": [
{"id": "1", "description": "Additional 1", "url": "A_Url 1", "resource_type": "documentation"},
{"id": "2", "description": "", "url": "", "resource_type": "documentation"},
{"id": "3", "description": "Additional 3", "url": "A_Url 3", "resource_type": "documentation"},
],
"timeseries_resources": [
{
"id": "4",
"description": "Timeseries 1",
"url": "T_Url 1",
"date": "T_Date 1",
"resource_type": "file",
},
{"id": "5", "description": "", "url": "", "date": "", "resource_type": "file"},
{"id": "6", "description": "Timeseries 3", "url": "", "date": "T_Date 3", "resource_type": "file"},
],
}
flattened_data = flatten_dict(data)
ignored = {}
errors = {}
# Add errors that validation would pick up for the blank resources
errors[("additional_resources", 1, "url")] = ["not empty"]
errors[("timeseries_resources", 1, "url")] = ["not empty"]
# Add a real validation error
errors[("timeseries_resources", 2, "url")] = ["not empty"]
remove_blank_resources(("__after",), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
"additional_resources": [
{"id": "1", "description": "Additional 1", "url": "A_Url 1", "resource_type": "documentation"},
{"id": "3", "description": "Additional 3", "url": "A_Url 3", "resource_type": "documentation"},
],
"timeseries_resources": [
{
"id": "4",
"description": "Timeseries 1",
"url": "T_Url 1",
"date": "T_Date 1",
"resource_type": "file",
},
{"id": "6", "description": "Timeseries 3", "url": "", "date": "T_Date 3", "resource_type": "file"},
],
}
expected_errors = {("timeseries_resources", 2, "url"): ["not empty"]}
assert_equal(result_data, expected_data)
assert_equal(errors, expected_errors)
def test_mixture_of_resource_types(self):
"""
Test with a mixture of additional and individual resources
"""
data = {
"resources": [
{"description": "Description 1", "url": "Url1", "resource_type": "documentation"},
{"description": "Description 2", "url": "Url2", "resource_type": "api"},
{"description": "Description 3", "url": "Url3", "resource_type": "file"},
{"description": "Description 4", "url": "Url4", "resource_type": "documentation"},
]
}
flattened_data = flatten_dict(data)
ignored = {}
errors = dict([(k, []) for k in flattened_data.keys()])
unmerge_resources(("__after",), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
"additional_resources": [
{"description": "Description 1", "url": "Url1", "resource_type": "documentation"},
{"description": "Description 4", "url": "Url4", "resource_type": "documentation"},
],
"individual_resources": [
{"description": "Description 2", "url": "Url2", "resource_type": "api"},
{"description": "Description 3", "url": "Url3", "resource_type": "file"},
],
}
expected_data.update(data)
assert_equal(result_data, expected_data)
def test_just_individual_resources(self):
"""
Tests with just one set of individual resources defined.
"""
data = {
"resources": [
{"description": "Description 1", "url": "Url1", "resource_type": "api", "date": ""},
{"description": "Description 2", "url": "Url2", "resource_type": "api"},
]
}
flattened_data = flatten_dict(data)
ignored = {}
errors = dict([(k, []) for k in flattened_data.keys()])
unmerge_resources(("__after",), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
"individual_resources": [
{"description": "Description 1", "url": "Url1", "resource_type": "api", "date": ""},
{"description": "Description 2", "url": "Url2", "resource_type": "api"},
]
}
expected_data.update(data)
assert_equal(result_data, expected_data)
def test_just_additional_resources(self):
"""
Tests with just one set of additional resources defined.
"""
data = {
'resources': [
{'description': 'Description 1', 'url': 'Url1', 'resource_type': 'documentation'},
{'description': 'Description 2', 'url': 'Url2', 'resource_type': 'documentation'}
]
}
flattened_data = flatten_dict(data)
ignored = {}
errors = dict([(k, []) for k in flattened_data.keys()])
unmerge_resources(('__after',), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
'additional_resources': [
{'description': 'Description 1', 'url': 'Url1', 'resource_type': 'documentation'},
{'description': 'Description 2', 'url': 'Url2', 'resource_type': 'documentation'}
]
}
expected_data.update(data)
assert_equal(result_data, expected_data)
def test_with_two_resource_types_defined(self):
"""
Tests with two resource types defined.
"""
data = {
'additional_resources': [
{'description': 'Additional 1', 'url': 'A_Url 1'},
{'description': 'Additional 2', 'url': 'A_Url 2'},
],
'timeseries_resources': [
{'description': 'Timeseries 1', 'url': 'T_Url 1', 'date': 'T_Date 1'},
{'description': 'Timeseries 2', 'url': 'T_Url 2', 'date': 'T_Date 2'},
{'description': 'Timeseries 3', 'url': 'T_Url 3', 'date': 'T_Date 3'},
]
}
flattened_data = flatten_dict(data)
ignored = {}
errors = dict([(k, []) for k in flattened_data.keys()])
merge_resources(('__after',), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
'resources': [
{'description': 'Additional 1', 'url': 'A_Url 1'},
{'description': 'Additional 2', 'url': 'A_Url 2'},
{'description': 'Timeseries 1', 'url': 'T_Url 1', 'date': 'T_Date 1'},
{'description': 'Timeseries 2', 'url': 'T_Url 2', 'date': 'T_Date 2'},
{'description': 'Timeseries 3', 'url': 'T_Url 3', 'date': 'T_Date 3'},
]
}
assert_equal(result_data, expected_data)
def validator(key, data, errors, context):
# if there was an error before calling our validator
# don't bother with our validation
if errors[key]:
return
try:
data_dict = df.unflatten(data[('__junk',)])
value = data_dict[key[0]]
if value is not missing:
if isinstance(value, basestring):
value = [value]
elif not isinstance(value, list):
errors[key].append(
_('Expecting list of strings, got "%s"') % str(value)
)
return
else:
value = []
if not errors[key]:
data[key] = json.dumps(value)
# remove from junk
del data_dict[key[0]]
data[('__junk',)] = df.flatten_dict(data_dict)
except KeyError:
pass
def test_merging_does_not_occur_when_there_have_been_validation_errors(
self):
"""
Test resources aren't merged when there have been other validation errors.
This is so that we don't throw out the user's original input.
"""
data = {
'additional_resources': [
{
'description': 'Description 1',
'url': 'Url 1'
},
{
'description': 'Description 2',
'url': 'Url 2'
},
]
}
flattened_data = flatten_dict(data)
errors = {('some_error', ): ['Oh dear, you did something wrong!']}
ignored = {}
merge_resources(('__after', ), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
assert_equal(data, result_data)
def test_mixture_of_resource_types(self):
"""
Test with a mixture of additional and individual resources
"""
data = {
'resources': [
{'description': 'Description 1', 'url': 'Url1', 'resource_type': 'documentation'},
{'description': 'Description 2', 'url': 'Url2', 'resource_type': 'api'},
{'description': 'Description 3', 'url': 'Url3', 'resource_type': 'file'},
{'description': 'Description 4', 'url': 'Url4', 'resource_type': 'documentation'}
]
}
flattened_data = flatten_dict(data)
ignored = {}
errors = dict([(k, []) for k in flattened_data.keys()])
unmerge_resources(('__after',), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
'additional_resources': [
{'description': 'Description 1', 'url': 'Url1', 'resource_type': 'documentation'},
{'description': 'Description 4', 'url': 'Url4', 'resource_type': 'documentation'}
],
'individual_resources': [
{'description': 'Description 2', 'url': 'Url2', 'resource_type': 'api'},
{'description': 'Description 3', 'url': 'Url3', 'resource_type': 'file'},
]
}
expected_data.update(data)
assert_equal(result_data, expected_data)
def test_visibility_validator_restricted_false(self):
tests = [
({
'private': True,
'visibility': 'private'
}, True),
({
'private': False,
'visibility': 'private'
}, True),
({
'private': True,
'visibility': 'restricted'
}, False),
({
'private': False,
'visibility': 'restricted'
}, False),
({
'private': True,
'visibility': 'public'
}, False),
({
'private': False,
'visibility': 'public'
}, False),
]
for test in tests:
key = ('visibility', )
data = df.flatten_dict(test[0])
validators.visibility_validator(key, data, {}, {})
assert (data[('private', )] == test[1]
), 'Data: {}, expected: {}, returned: {}'.format(
test[0], test[1], data[('private', )])
def test_check_direct_download_url_invalid(self):
errors = defaultdict(list)
dada = copy.deepcopy(self.test_data)
dada['availability'] = u'direct_download'
dada['resources'][0]['url'] = u''
data = flatten_dict(dada)
self.assertRaises(Exception, check_direct_download_url, ('resources', 0, 'url'), data, errors, None)
def test_just_additional_resources(self):
"""
Tests with just one set of additional resources defined.
"""
data = {
'resources': [
{'description': 'Description 1', 'url': 'Url1', 'resource_type': 'documentation'},
{'description': 'Description 2', 'url': 'Url2', 'resource_type': 'documentation'}
]
}
flattened_data = flatten_dict(data)
ignored = {}
errors = dict([(k, []) for k in flattened_data.keys()])
unmerge_resources(('__after',), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
'additional_resources': [
{'description': 'Description 1', 'url': 'Url1', 'resource_type': 'documentation'},
{'description': 'Description 2', 'url': 'Url2', 'resource_type': 'documentation'}
]
}
expected_data.update(data)
assert_equal(result_data, expected_data)
def test_simple_case(self):
"""
Tests with just one set of resources defined.
"""
data = {
'additional_resources': [
{'description': 'Description 1', 'url': 'Url 1'},
{'description': 'Description 2', 'url': 'Url 2'},
]
}
flattened_data = flatten_dict(data)
ignored = {}
errors = dict([(k, []) for k in flattened_data.keys()])
merge_resources(('__after',), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
'resources': [
{'description': 'Description 1', 'url': 'Url 1'},
{'description': 'Description 2', 'url': 'Url 2'},
]
}
assert_equal(result_data, expected_data)
def validator(key, data, errors, context):
# if there was an error before calling our validator
# don't bother with our validation
if errors[key]:
return
try:
data_dict = df.unflatten(data[HARVEST_JUNK])
value = data_dict[key[0]]
if value is not missing:
if isinstance(value, basestring):
value = [value]
elif not isinstance(value, list):
errors[key].append(
_('Expecting list of strings, got "%s"') % str(value))
return
else:
value = []
if not errors[key]:
data[key] = json.dumps(value)
# remove from junk
del data_dict[key[0]]
data[HARVEST_JUNK] = df.flatten_dict(data_dict)
except KeyError:
pass
def test_mixture_of_resource_types(self):
"""
Test with a mixture of additional and individual resources
"""
data = {
'resources': [
{'description': 'Description 1', 'url': 'Url1', 'resource_type': 'documentation'},
{'description': 'Description 2', 'url': 'Url2', 'resource_type': 'api'},
{'description': 'Description 3', 'url': 'Url3', 'resource_type': 'file'},
{'description': 'Description 4', 'url': 'Url4', 'resource_type': 'documentation'}
]
}
flattened_data = flatten_dict(data)
ignored = {}
errors = dict([(k, []) for k in flattened_data.keys()])
unmerge_resources(('__after',), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
'additional_resources': [
{'description': 'Description 1', 'url': 'Url1', 'resource_type': 'documentation'},
{'description': 'Description 4', 'url': 'Url4', 'resource_type': 'documentation'}
],
'individual_resources': [
{'description': 'Description 2', 'url': 'Url2', 'resource_type': 'api'},
{'description': 'Description 3', 'url': 'Url3', 'resource_type': 'file'},
]
}
expected_data.update(data)
assert_equal(result_data, expected_data)
def test_if_empty_guess_format(self):
data = {'name': 'package_name', 'resources': [
{'url': 'http://fakedomain/my.csv', 'format': ''},
{'url': 'http://fakedomain/my.pdf',
'format': df.Missing},
{'url': 'http://fakedomain/my.pdf', 'format': 'pdf'},
{'url': 'http://fakedomain/my.pdf',
'id': 'fake_resource_id', 'format': ''}
]}
data = df.flatten_dict(data)
@t.does_not_modify_errors_dict
def call_validator(*args, **kwargs):
return validators.if_empty_guess_format(*args, **kwargs)
new_data = copy.deepcopy(data)
call_validator(key=('resources', 0, 'format'), data=new_data,
errors={}, context={})
assert new_data[('resources', 0, 'format')] == 'text/csv'
new_data = copy.deepcopy(data)
call_validator(key=('resources', 1, 'format'), data=new_data,
errors={}, context={})
assert new_data[('resources', 1, 'format')] == 'application/pdf'
new_data = copy.deepcopy(data)
call_validator(key=('resources', 2, 'format'), data=new_data,
errors={}, context={})
assert new_data[('resources', 2, 'format')] == 'pdf'
new_data = copy.deepcopy(data)
call_validator(key=('resources', 3, 'format'), data=new_data,
errors={}, context={})
assert new_data[('resources', 3, 'format')] == ''
def test_with_two_resource_types_defined(self):
"""
Tests with two resource types defined.
"""
data = {
'additional_resources': [
{'description': 'Additional 1', 'url': 'A_Url 1'},
{'description': 'Additional 2', 'url': 'A_Url 2'},
],
'timeseries_resources': [
{'description': 'Timeseries 1', 'url': 'T_Url 1', 'date': 'T_Date 1'},
{'description': 'Timeseries 2', 'url': 'T_Url 2', 'date': 'T_Date 2'},
{'description': 'Timeseries 3', 'url': 'T_Url 3', 'date': 'T_Date 3'},
]
}
flattened_data = flatten_dict(data)
ignored = {}
errors = dict([(k, []) for k in flattened_data.keys()])
merge_resources(('__after',), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
'resources': [
{'description': 'Additional 1', 'url': 'A_Url 1'},
{'description': 'Additional 2', 'url': 'A_Url 2'},
{'description': 'Timeseries 1', 'url': 'T_Url 1', 'date': 'T_Date 1'},
{'description': 'Timeseries 2', 'url': 'T_Url 2', 'date': 'T_Date 2'},
{'description': 'Timeseries 3', 'url': 'T_Url 3', 'date': 'T_Date 3'},
]
}
assert_equal(result_data, expected_data)
def test_simple_case(self):
"""
Tests with just one set of resources defined.
"""
data = {
'additional_resources': [
{'description': 'Description 1', 'url': 'Url 1'},
{'description': 'Description 2', 'url': 'Url 2'},
]
}
flattened_data = flatten_dict(data)
ignored = {}
errors = dict([(k, []) for k in flattened_data.keys()])
merge_resources(('__after',), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
'resources': [
{'description': 'Description 1', 'url': 'Url 1'},
{'description': 'Description 2', 'url': 'Url 2'},
]
}
assert_equal(result_data, expected_data)
def test_simple_case(self):
"""
Tests with just one set of resources defined.
"""
data = {
"additional_resources": [
{"description": "Description 1", "url": "Url 1"},
{"description": "Description 2", "url": "Url 2"},
]
}
flattened_data = flatten_dict(data)
ignored = {}
errors = dict([(k, []) for k in flattened_data.keys()])
merge_resources(("__after",), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
"resources": [
{"description": "Description 1", "url": "Url 1"},
{"description": "Description 2", "url": "Url 2"},
]
}
assert_equal(result_data, expected_data)
def test_flatten_no_extra():
data = {'foo': 'bar', 'foo2': 'bar2'}
flattened_data = flatten_dict(data)
assert flattened_data == {('foo', ): u'bar', ('foo2', ): u'bar2'}
assert data == unflatten(flattened_data)
def test_flatten():
data = {'extras': [{'key': 'genre', 'value': u'horror'},
{'key': 'media', 'value': u'dvd'}],
'license_id': u'gpl-3.0',
'name': u'testpkg',
'resources': [{u'alt_url': u'alt_url',
u'description': u'Second file',
u'extras': {u'size': u'200'},
u'format': u'xml',
u'hash': u'def123',
u'url': u'http://blah.com/file2.xml'},
{u'alt_url': u'alt_url',
u'description': u'Main file',
u'extras': {u'size': u'200'},
u'format': u'xml',
u'hash': u'abc123',
u'url': u'http://blah.com/file.xml'}],
'tags': [{'name': u'russion'}, {'name': u'novel'}],
'title': u'Some Title',
'url': u'http://blahblahblah.mydomain'}
assert flatten_dict(data) == {('extras', 0, 'key'): 'genre',
('extras', 0, 'value'): u'horror',
('extras', 1, 'key'): 'media',
('extras', 1, 'value'): u'dvd',
('license_id',): u'gpl-3.0',
('name',): u'testpkg',
('resources', 0, u'alt_url'): u'alt_url',
('resources', 0, u'description'): u'Second file',
('resources', 0, u'extras'): {u'size': u'200'},
('resources', 0, u'format'): u'xml',
('resources', 0, u'hash'): u'def123',
('resources', 0, u'url'): u'http://blah.com/file2.xml',
('resources', 1, u'alt_url'): u'alt_url',
('resources', 1, u'description'): u'Main file',
('resources', 1, u'extras'): {u'size': u'200'},
('resources', 1, u'format'): u'xml',
('resources', 1, u'hash'): u'abc123',
('resources', 1, u'url'): u'http://blah.com/file.xml',
('tags', 0, 'name'): u'russion',
('tags', 1, 'name'): u'novel',
('title',): u'Some Title',
('url',): u'http://blahblahblah.mydomain'}, pformat(flatten_dict(data))
assert data == unflatten(flatten_dict(data))
def test_validate_mimetype_invalid(self):
errors = defaultdict(list)
data_dict = copy.deepcopy(self.test_data)
data_dict['resources'][0]['format'] = u'application/pdf><'
data = flatten_dict(data_dict)
self.assertRaises(Invalid, validate_mimetype, ('resources', 0, 'format',), data, errors, None)
def test_validate_algorithm_invalid(self):
errors = defaultdict(list)
data_dict = copy.deepcopy(self.test_data)
data_dict['resources'][0]['algorithm'] = u'RadioGatún-1216!>'
data = flatten_dict(data_dict)
self.assertRaises(Invalid, validate_algorithm, ('resources', 0, 'algorithm',), data, errors, None)
def test_flatten():
data = {'extras': [{'key': 'genre', 'value': u'horror'},
{'key': 'media', 'value': u'dvd'}],
'license_id': u'gpl-3.0',
'name': u'testpkg',
'resources': [{u'alt_url': u'alt_url',
u'description': u'Second file',
u'extras': {u'size': u'200'},
u'format': u'xml',
u'hash': u'def123',
u'url': u'http://blah.com/file2.xml'},
{u'alt_url': u'alt_url',
u'description': u'Main file',
u'extras': {u'size': u'200'},
u'format': u'xml',
u'hash': u'abc123',
u'url': u'http://blah.com/file.xml'}],
'tags': [{'name': u'russion'}, {'name': u'novel'}],
'title': u'Some Title',
'url': u'http://blahblahblah.mydomain'}
assert flatten_dict(data) == {('extras', 0, 'key'): 'genre',
('extras', 0, 'value'): u'horror',
('extras', 1, 'key'): 'media',
('extras', 1, 'value'): u'dvd',
('license_id',): u'gpl-3.0',
('name',): u'testpkg',
('resources', 0, u'alt_url'): u'alt_url',
('resources', 0, u'description'): u'Second file',
('resources', 0, u'extras'): {u'size': u'200'},
('resources', 0, u'format'): u'xml',
('resources', 0, u'hash'): u'def123',
('resources', 0, u'url'): u'http://blah.com/file2.xml',
('resources', 1, u'alt_url'): u'alt_url',
('resources', 1, u'description'): u'Main file',
('resources', 1, u'extras'): {u'size': u'200'},
('resources', 1, u'format'): u'xml',
('resources', 1, u'hash'): u'abc123',
('resources', 1, u'url'): u'http://blah.com/file.xml',
('tags', 0, 'name'): u'russion',
('tags', 1, 'name'): u'novel',
('title',): u'Some Title',
('url',): u'http://blahblahblah.mydomain'}, pformat(flatten_dict(data))
assert data == unflatten(flatten_dict(data))
def test_flatten_empty():
data = {}
flattened_data = flatten_dict(data)
assert flattened_data == {}
assert data == unflatten(flattened_data)
def test_flatten_empty():
data = {}
flattened_data = flatten_dict(data)
assert flattened_data == {}
assert data == unflatten(flattened_data)
def test_check_direct_download_url_invalid(self):
errors = defaultdict(list)
dada = copy.deepcopy(self.test_data)
dada['availability'] = u'direct_download'
dada['resources'][0]['url'] = u''
data = flatten_dict(dada)
self.assertRaises(Exception,
check_resource_url_for_direct_download_url,
('resources', 0, 'url'), data, errors, None)
def test_flatten_no_extra():
data = {'foo':'bar',
'foo2':'bar2'}
flattened_data = flatten_dict(data)
assert flattened_data == {('foo',): u'bar', ('foo2', ): u'bar2'}
assert data == unflatten(flattened_data)
def test_if_empty_guess_format():
data = {
"name": "package_name",
"resources": [
{"url": "http://fakedomain/my.csv", "format": ""},
{"url": "http://fakedomain/my.pdf", "format": df.Missing},
{"url": "http://fakedomain/my.pdf", "format": "pdf"},
{
"url": "http://fakedomain/my.pdf",
"id": "fake_resource_id",
"format": "",
},
{"url": "http://example.com", "format": ""},
{"url": "my.csv", "format": ""}
],
}
data = df.flatten_dict(data)
@t.does_not_modify_errors_dict
def call_validator(*args, **kwargs):
return validators.if_empty_guess_format(*args, **kwargs)
new_data = copy.deepcopy(data)
call_validator(
key=("resources", 0, "format"), data=new_data, errors={}, context={}
)
assert new_data[("resources", 0, "format")] == "text/csv"
new_data = copy.deepcopy(data)
call_validator(
key=("resources", 1, "format"), data=new_data, errors={}, context={}
)
assert new_data[("resources", 1, "format")] == "application/pdf"
new_data = copy.deepcopy(data)
call_validator(
key=("resources", 2, "format"), data=new_data, errors={}, context={}
)
assert new_data[("resources", 2, "format")] == "pdf"
new_data = copy.deepcopy(data)
call_validator(
key=("resources", 3, "format"), data=new_data, errors={}, context={}
)
assert new_data[("resources", 3, "format")] == ""
new_data = copy.deepcopy(data)
call_validator(
key=("resources", 4, "format"), data=new_data, errors={}, context={}
)
assert new_data[("resources", 4, "format")] == ""
new_data = copy.deepcopy(data)
call_validator(
key=("resources", 5, "format"), data=new_data, errors={}, context={}
)
assert new_data[("resources", 5, "format")] == "text/csv"
def test_flatten_deeper():
data = {
u"resources": [
{
u"subfields": [
{
u"test": u"hello",
},
],
},
],
}
assert flatten_dict(data) == {
("resources", 0, u"subfields", 0, u"test"): u"hello",
}, pformat(flatten_dict(data))
assert data == unflatten(flatten_dict(data)), pformat(
unflatten(flatten_dict(data)))
def validator(key, data, errors, context):
if 'package' not in context:
return
data_dict = flatten_dict(get_action('package_show')(context, {'id': context['package'].id}))
if key not in data or data[key] is missing:
if key in data_dict:
data[key] = data_dict[key]
def validator(key, data, errors, context):
if 'package' not in context:
return
data_dict = flatten_dict(get_action('package_show')(context, {'id': context['package'].id}))
if key not in data or data[key] is missing:
if key in data_dict:
data[key] = data_dict[key]
def test_validate_algorithm_valid(self):
errors = defaultdict(list)
data_dict = copy.deepcopy(self.test_data)
data_dict['resources'][0]['algorithm'] = u'RadioGatún-1216'
data = flatten_dict(data_dict)
try:
validate_algorithm(('resources', 0, 'algorithm',), data, errors, None)
except Invalid:
raise AssertionError('Algorithm raised exception, it should not')
def test_validate_mimetype_valid(self):
errors = defaultdict(list)
data_dict = copy.deepcopy(self.test_data)
data_dict['resources'][0]['format'] = u'vnd.3gpp2.bcmcsinfo+xml/'
# flatten dict (or change test_data to flattened form?)
data = flatten_dict(data_dict)
try:
validate_mimetype(('resources', 0, 'mimetype',), data, errors, None)
except Invalid:
raise AssertionError('Mimetype raised exception, it should not')
def test_validate_mimetype_invalid(self):
errors = defaultdict(list)
data_dict = copy.deepcopy(self.test_data)
data_dict['resources'][0]['format'] = u'application/pdf><'
data = flatten_dict(data_dict)
self.assertRaises(Invalid, validate_mimetype, (
'resources',
0,
'format',
), data, errors, None)
def test_validate_algorithm_invalid(self):
errors = defaultdict(list)
data_dict = copy.deepcopy(self.test_data)
data_dict['resources'][0]['algorithm'] = u'RadioGatún-1216!>'
data = flatten_dict(data_dict)
self.assertRaises(Invalid, validate_algorithm, (
'resources',
0,
'algorithm',
), data, errors, None)
def converter(key, data, errors, context):
single_value = False
junk_key = ('__junk', )
junk = unflatten(data.get(junk_key, {}))
# for multiple-valued fields, everything moved to junk
sub_data = data.get(key)
if not sub_data or sub_data is missing:
sub_data = junk
try:
for k in key[:-1]:
sub_data = sub_data[k]
sub_data = sub_data.pop(key[-1], None)
except (KeyError, IndexError):
sub_data = None
if not sub_data or sub_data is missing:
data[key] = missing
return
data[junk_key] = flatten_dict(junk)
schema = getattr(sub_schema, 'get_default_{}_schema'.format(name))()
if not isinstance(sub_data, (list, dict)):
try:
sub_data = json.loads(sub_data)
except ValueError:
raise Invalid(_('Plain values are not supported'))
if key[-1:] in incorrectly_dictized_dict:
try:
sub_data = sub_data[0]
except KeyError:
pass
if isinstance(sub_data, dict):
single_value = True
sub_data = [sub_data]
sub_data = [_listize(item) for item in sub_data]
validated_list, errors_list = _validate_sub_data(
sub_data, schema, context
)
data[key] = validated_list[0] if single_value else validated_list
if any(err for err in errors_list):
errors.setdefault(key, []).extend(errors_list)
raise StopOnError
def test_validate_mimetype_valid(self):
errors = defaultdict(list)
data_dict = copy.deepcopy(self.test_data)
data_dict['resources'][0]['format'] = u'vnd.3gpp2.bcmcsinfo+xml/'
# flatten dict (or change test_data to flattened form?)
data = flatten_dict(data_dict)
try:
validate_mimetype((
'resources',
0,
'mimetype',
), data, errors, None)
except Invalid:
raise AssertionError('Mimetype raised exception, it should not')
def test_validate_pid_uniqueness_4(self):
errors = defaultdict(list)
data = self._get_unflattened_data()
self._set_unflattened_data_random_id(data)
flat_data = flatten_dict(copy.deepcopy(data))
self._set_random_pids_for_unflattened_data(data)
self._create_package_with_unflattened_data(data)
try:
validate_pid_uniqueness(('pids', 0, 'id'), flat_data, errors, None)
validate_pid_uniqueness(('pids', 1, 'id'), flat_data, errors, None)
validate_pid_uniqueness(('pids', 2, 'id'), flat_data, errors, None)
validate_pid_uniqueness(('pids', 3, 'id'), flat_data, errors, None)
except Invalid as e:
self.fail("validate_pid_uniqueness_4 failed: {0}".format(e))
def test_validate_algorithm_valid(self):
errors = defaultdict(list)
data_dict = copy.deepcopy(self.test_data)
data_dict['resources'][0]['algorithm'] = u'RadioGatún-1216'
data = flatten_dict(data_dict)
try:
validate_algorithm((
'resources',
0,
'algorithm',
), data, errors, None)
except Invalid:
raise AssertionError('Algorithm raised exception, it should not')
def validator(key, data, errors, context):
if 'package' not in context:
return
data_dict = flatten_dict(
get_action('package_show')(context, {
'id': context['package'].id
}))
if not authz.is_authorized('sysadmin', context).get('success'):
if key in data_dict:
data[key] = data_dict[key]
else:
del data[key]
def test_remove_blank_resources(self):
"""
Blank rows occur as a result of switching to/from timeseries resource
or deleting a resource in a table (you can't remove the row itself).
Therefore remove blank rows.
"""
import nose
nose.tools.maxDiff = 2000
data = {
'additional_resources': [
{'id': '1', 'description': 'Additional 1', 'url': 'A_Url 1', 'resource_type': 'documentation'},
{'id': '2', 'description': '', 'url': '', 'resource_type': 'documentation'},
{'id': '3', 'description': 'Additional 3', 'url': 'A_Url 3', 'resource_type': 'documentation'},
],
'timeseries_resources': [
{'id': '4', 'description': 'Timeseries 1', 'url': 'T_Url 1', 'date': 'T_Date 1', 'resource_type': 'file'},
{'id': '5', 'description': '', 'url': '', 'date': '', 'resource_type': 'file'},
{'id': '6', 'description': 'Timeseries 3', 'url': '', 'date': 'T_Date 3', 'resource_type': 'file'},
]
}
flattened_data = flatten_dict(data)
ignored = {}
errors = {}
# Add errors that validation would pick up for the blank resources
errors[('additional_resources', 1, 'url')] = ['not empty']
errors[('timeseries_resources', 1, 'url')] = ['not empty']
# Add a real validation error
errors[('timeseries_resources', 2, 'url')] = ['not empty']
remove_blank_resources(('__after',), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
'additional_resources': [
{'id': '1', 'description': 'Additional 1', 'url': 'A_Url 1', 'resource_type': 'documentation'},
{'id': '3', 'description': 'Additional 3', 'url': 'A_Url 3', 'resource_type': 'documentation'},
],
'timeseries_resources': [
{'id': '4', 'description': 'Timeseries 1', 'url': 'T_Url 1', 'date': 'T_Date 1', 'resource_type': 'file'},
{'id': '6', 'description': 'Timeseries 3', 'url': '', 'date': 'T_Date 3', 'resource_type': 'file'},
]
}
expected_errors = {('timeseries_resources', 2, 'url'): ['not empty']}
assert_equal(result_data, expected_data)
assert_equal(errors, expected_errors)
def test_remove_blank_resources(self):
"""
Blank rows occur as a result of switching to/from timeseries resource
or deleting a resource in a table (you can't remove the row itself).
Therefore remove blank rows.
"""
import nose
nose.tools.maxDiff = 2000
data = {
'additional_resources': [
{'id': '1', 'description': 'Additional 1', 'url': 'A_Url 1', 'resource_type': 'documentation'},
{'id': '2', 'description': '', 'url': '', 'resource_type': 'documentation'},
{'id': '3', 'description': 'Additional 3', 'url': 'A_Url 3', 'resource_type': 'documentation'},
],
'timeseries_resources': [
{'id': '4', 'description': 'Timeseries 1', 'url': 'T_Url 1', 'date': 'T_Date 1', 'resource_type': 'file'},
{'id': '5', 'description': '', 'url': '', 'date': '', 'resource_type': 'file'},
{'id': '6', 'description': 'Timeseries 3', 'url': '', 'date': 'T_Date 3', 'resource_type': 'file'},
]
}
flattened_data = flatten_dict(data)
ignored = {}
errors = {}
# Add errors that validation would pick up for the blank resources
errors[('additional_resources', 1, 'url')] = ['not empty']
errors[('timeseries_resources', 1, 'url')] = ['not empty']
# Add a real validation error
errors[('timeseries_resources', 2, 'url')] = ['not empty']
remove_blank_resources(('__after',), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
'additional_resources': [
{'id': '1', 'description': 'Additional 1', 'url': 'A_Url 1', 'resource_type': 'documentation'},
{'id': '3', 'description': 'Additional 3', 'url': 'A_Url 3', 'resource_type': 'documentation'},
],
'timeseries_resources': [
{'id': '4', 'description': 'Timeseries 1', 'url': 'T_Url 1', 'date': 'T_Date 1', 'resource_type': 'file'},
{'id': '6', 'description': 'Timeseries 3', 'url': '', 'date': 'T_Date 3', 'resource_type': 'file'},
]
}
expected_errors = {('timeseries_resources', 2, 'url'): ['not empty']}
assert_equal(result_data, expected_data)
assert_equal(errors, expected_errors)
def test_error_keys_are_unmerged(self):
"""
Test that the items in errors are unmerged too.
Note - errors may contain key values not present in the data
"""
data = {
'resources': [
# additional resources
{'description': 'Additional 1', 'url': 'A_Url 1', 'resource_type': 'documentation'},
{'description': 'Additional 2', 'url': 'A_Url 2', 'resource_type': 'documentation'},
# individual resources
{'description': 'Individual 1', 'url': 'I_Url 1', 'resource_type': 'api'},
{'description': 'Individual 2', 'url': 'I_Url 2', 'resource_type': 'api'},
{'description': 'Individual 3', 'url': 'I_Url 3', 'resource_type': 'api'},
],
}
flattened_data = flatten_dict(data)
errors = dict([(k, []) for k in flattened_data.keys()])
# Add some extra fields into errors
errors[('resources', 0, 'foo')] = []
errors[('resources', 1, 'foo')] = []
errors[('resources', 2, 'foo')] = []
errors[('resources', 3, 'foo')] = []
errors[('resources', 4, 'foo')] = []
ignored = {}
unmerge_resources(('__after',), flattened_data, errors, ignored)
result_errors = unflatten(errors)
expected_errors = {
'additional_resources': [
{'description': [], 'url': [], 'resource_type': [], 'foo': []},
{'description': [], 'url': [], 'resource_type': [], 'foo': []},
],
'individual_resources': [
{'description': [], 'url': [], 'resource_type': [], 'foo': []},
{'description': [], 'url': [], 'resource_type': [], 'foo': []},
{'description': [], 'url': [], 'resource_type': [], 'foo': []},
]
}
assert_equal(result_errors, expected_errors)
def test_error_keys_are_unmerged(self):
"""
Test that the items in errors are unmerged too.
Note - errors may contain key values not present in the data
"""
data = {
'resources': [
# additional resources
{'description': 'Additional 1', 'url': 'A_Url 1', 'resource_type': 'documentation'},
{'description': 'Additional 2', 'url': 'A_Url 2', 'resource_type': 'documentation'},
# individual resources
{'description': 'Individual 1', 'url': 'I_Url 1', 'resource_type': 'api'},
{'description': 'Individual 2', 'url': 'I_Url 2', 'resource_type': 'api'},
{'description': 'Individual 3', 'url': 'I_Url 3', 'resource_type': 'api'},
],
}
flattened_data = flatten_dict(data)
errors = dict([(k, []) for k in flattened_data.keys()])
# Add some extra fields into errors
errors[('resources', 0, 'foo')] = []
errors[('resources', 1, 'foo')] = []
errors[('resources', 2, 'foo')] = []
errors[('resources', 3, 'foo')] = []
errors[('resources', 4, 'foo')] = []
ignored = {}
unmerge_resources(('__after',), flattened_data, errors, ignored)
result_errors = unflatten(errors)
expected_errors = {
'additional_resources': [
{'description': [], 'url': [], 'resource_type': [], 'foo': []},
{'description': [], 'url': [], 'resource_type': [], 'foo': []},
],
'individual_resources': [
{'description': [], 'url': [], 'resource_type': [], 'foo': []},
{'description': [], 'url': [], 'resource_type': [], 'foo': []},
{'description': [], 'url': [], 'resource_type': [], 'foo': []},
]
}
assert_equal(result_errors, expected_errors)
def test_error_keys_are_unmerged(self):
"""
Test that the items in errors are unmerged too.
Note - errors may contain key values not present in the data
"""
data = {
"resources": [
# additional resources
{"description": "Additional 1", "url": "A_Url 1", "resource_type": "documentation"},
{"description": "Additional 2", "url": "A_Url 2", "resource_type": "documentation"},
# individual resources
{"description": "Individual 1", "url": "I_Url 1", "resource_type": "api"},
{"description": "Individual 2", "url": "I_Url 2", "resource_type": "api"},
{"description": "Individual 3", "url": "I_Url 3", "resource_type": "api"},
]
}
flattened_data = flatten_dict(data)
errors = dict([(k, []) for k in flattened_data.keys()])
# Add some extra fields into errors
errors[("resources", 0, "foo")] = []
errors[("resources", 1, "foo")] = []
errors[("resources", 2, "foo")] = []
errors[("resources", 3, "foo")] = []
errors[("resources", 4, "foo")] = []
ignored = {}
unmerge_resources(("__after",), flattened_data, errors, ignored)
result_errors = unflatten(errors)
expected_errors = {
"additional_resources": [
{"description": [], "url": [], "resource_type": [], "foo": []},
{"description": [], "url": [], "resource_type": [], "foo": []},
],
"individual_resources": [
{"description": [], "url": [], "resource_type": [], "foo": []},
{"description": [], "url": [], "resource_type": [], "foo": []},
{"description": [], "url": [], "resource_type": [], "foo": []},
],
}
assert_equal(result_errors, expected_errors)
def test_just_timeseries_resources(self):
"""
Tests with just one set of timeseries resources defined.
"""
data = {
'resources': [
{
'description': 'Description 1',
'url': 'Url1',
'resource_type': 'api',
'date': '2011-12-25',
},
{
'description': 'Description 2',
'url': 'Url2',
'resource_type': 'api',
'date': '2011-11-25'
},
]
}
flattened_data = flatten_dict(data)
ignored = {}
errors = dict([(k, []) for k in flattened_data.keys()])
unmerge_resources(('__after', ), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
expected_data = {
'timeseries_resources': [{
'description': 'Description 1',
'url': 'Url1',
'resource_type': 'api',
'date': '2011-12-25'
}, {
'description': 'Description 2',
'url': 'Url2',
'resource_type': 'api',
'date': '2011-11-25'
}]
}
expected_data.update(data)
assert_equal(result_data, expected_data)
def test_error_keys_are_merged_over(self):
"""
Test that the items in errors are merged over correctly too.
Note - errors may contain key values not present in the data
"""
data = {
'additional_resources': [
{'description': 'Additional 1', 'url': 'A_Url 1'},
{'description': 'Additional 2', 'url': 'A_Url 2'},
],
'timeseries_resources': [
{'description': 'Timeseries 1', 'url': 'T_Url 1', 'date': 'T_Date 1'},
{'description': 'Timeseries 2', 'url': 'T_Url 2', 'date': 'T_Date 2'},
{'description': 'Timeseries 3', 'url': 'T_Url 3', 'date': 'T_Date 3'},
]
}
flattened_data = flatten_dict(data)
errors = dict([(k, []) for k in flattened_data.keys()])
# Add some extra fields into errors
errors[('additional_resources', 0, 'resource_type')] = []
errors[('additional_resources', 1, 'resource_type')] = []
errors[('timeseries_resources', 0, 'resource_type')] = []
errors[('timeseries_resources', 1, 'resource_type')] = []
errors[('timeseries_resources', 2, 'resource_type')] = []
ignored = {}
merge_resources(('__after',), flattened_data, errors, ignored)
result_errors = unflatten(errors)
expected_errors = {
'resources': [
{'description': [], 'url': [], 'resource_type': []},
{'description': [], 'url': [], 'resource_type': []},
{'description': [], 'url': [], 'date': [], 'resource_type': []},
{'description': [], 'url': [], 'date': [], 'resource_type': []},
{'description': [], 'url': [], 'date': [], 'resource_type': []},
]
}
assert_equal(result_errors, expected_errors)
def test_error_keys_are_merged_over(self):
"""
Test that the items in errors are merged over correctly too.
Note - errors may contain key values not present in the data
"""
data = {
'additional_resources': [
{'description': 'Additional 1', 'url': 'A_Url 1'},
{'description': 'Additional 2', 'url': 'A_Url 2'},
],
'timeseries_resources': [
{'description': 'Timeseries 1', 'url': 'T_Url 1', 'date': 'T_Date 1'},
{'description': 'Timeseries 2', 'url': 'T_Url 2', 'date': 'T_Date 2'},
{'description': 'Timeseries 3', 'url': 'T_Url 3', 'date': 'T_Date 3'},
]
}
flattened_data = flatten_dict(data)
errors = dict([(k, []) for k in flattened_data.keys()])
# Add some extra fields into errors
errors[('additional_resources', 0, 'resource_type')] = []
errors[('additional_resources', 1, 'resource_type')] = []
errors[('timeseries_resources', 0, 'resource_type')] = []
errors[('timeseries_resources', 1, 'resource_type')] = []
errors[('timeseries_resources', 2, 'resource_type')] = []
ignored = {}
merge_resources(('__after',), flattened_data, errors, ignored)
result_errors = unflatten(errors)
expected_errors = {
'resources': [
{'description': [], 'url': [], 'resource_type': []},
{'description': [], 'url': [], 'resource_type': []},
{'description': [], 'url': [], 'date': [], 'resource_type': []},
{'description': [], 'url': [], 'date': [], 'resource_type': []},
{'description': [], 'url': [], 'date': [], 'resource_type': []},
]
}
assert_equal(result_errors, expected_errors)
def test_error_keys_are_merged_over(self):
"""
Test that the items in errors are merged over correctly too.
Note - errors may contain key values not present in the data
"""
data = {
"additional_resources": [
{"description": "Additional 1", "url": "A_Url 1"},
{"description": "Additional 2", "url": "A_Url 2"},
],
"timeseries_resources": [
{"description": "Timeseries 1", "url": "T_Url 1", "date": "T_Date 1"},
{"description": "Timeseries 2", "url": "T_Url 2", "date": "T_Date 2"},
{"description": "Timeseries 3", "url": "T_Url 3", "date": "T_Date 3"},
],
}
flattened_data = flatten_dict(data)
errors = dict([(k, []) for k in flattened_data.keys()])
# Add some extra fields into errors
errors[("additional_resources", 0, "resource_type")] = []
errors[("additional_resources", 1, "resource_type")] = []
errors[("timeseries_resources", 0, "resource_type")] = []
errors[("timeseries_resources", 1, "resource_type")] = []
errors[("timeseries_resources", 2, "resource_type")] = []
ignored = {}
merge_resources(("__after",), flattened_data, errors, ignored)
result_errors = unflatten(errors)
expected_errors = {
"resources": [
{"description": [], "url": [], "resource_type": []},
{"description": [], "url": [], "resource_type": []},
{"description": [], "url": [], "date": [], "resource_type": []},
{"description": [], "url": [], "date": [], "resource_type": []},
{"description": [], "url": [], "date": [], "resource_type": []},
]
}
assert_equal(result_errors, expected_errors)
def test_merging_does_not_occur_when_there_have_been_validation_errors(self):
"""
Test resources aren't merged when there have been other validation errors.
This is so that we don't throw out the user's original input.
"""
data = {
"additional_resources": [
{"description": "Description 1", "url": "Url 1"},
{"description": "Description 2", "url": "Url 2"},
]
}
flattened_data = flatten_dict(data)
errors = {("some_error",): ["Oh dear, you did something wrong!"]}
ignored = {}
merge_resources(("__after",), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
assert_equal(data, result_data)
def test_visibility_validator_set_correct_value(self):
tests = [
({
'visibility': 'private'
}, 'restricted'),
({
'visibility': 'restricted'
}, 'restricted'),
({
'visibility': 'public'
}, 'public'),
]
for test in tests:
key = ('visibility', )
data = df.flatten_dict(test[0])
validators.visibility_validator(key, data, {}, {})
assert (data[('visibility', )] == test[1]
), 'Data: {}, expected: {}, returned: {}'.format(
test[0], test[1], data[('private', )])
def test_merging_does_not_occur_when_there_have_been_validation_errors(self):
"""
Test resources aren't merged when there have been other validation errors.
This is so that we don't throw out the user's original input.
"""
data = {
'additional_resources': [
{'description': 'Description 1', 'url': 'Url 1'},
{'description': 'Description 2', 'url': 'Url 2'},
]
}
flattened_data = flatten_dict(data)
errors = {('some_error',): ['Oh dear, you did something wrong!']}
ignored = {}
merge_resources(('__after',), flattened_data, errors, ignored)
result_data = unflatten(flattened_data)
assert_equal(data, result_data)
def test_validate_primary_pid_uniqueness(self):
errors = defaultdict(list)
unflattened_data = self._get_unflattened_data()
ppid_1 = u'primary_pid_1'
ppid_2 = u'primary_pid_2'
flat_data = flatten_dict(copy.deepcopy(unflattened_data))
unflattened_data['pids'][0]['id'] = ppid_1
flat_data[('pids',0,'id')] = ppid_2
self._set_unflattened_data_id(unflattened_data)
self._set_flat_data_id(flat_data)
# Validate against empty database == no other datasets with the same primary pid
validate_primary_pid_uniqueness(('pids',0,'id'), flat_data, errors, None)
assert len(errors) == 0
self._create_package_with_unflattened_data(unflattened_data)
# Validate against database containing one dataset that has different primary pid
validate_primary_pid_uniqueness(('pids',0,'id'), flat_data, errors, None)
assert len(errors) == 0
# Validate against database containing one dataset having the same primary pid
flat_data[('pids',0,'id')] = ppid_1
self.assertRaises(Invalid, validate_primary_pid_uniqueness, ('pids',0,'id'), flat_data, errors, None)
def test_validate_external_id_uniqueness(self):
errors = defaultdict(list)
unflattened_data = self._get_unflattened_data()
ext_id_1 = u'urn:nbn:fi:csc-ida9238572983572398s'
ext_id_2 = u'urn:nbn:fi:csc-ida9238572983572399s'
flat_data = flatten_dict(copy.deepcopy(unflattened_data))
unflattened_data['external_id'] = ext_id_1
flat_data[('external_id', )] = ext_id_2
self._set_unflattened_data_id(unflattened_data)
self._set_flat_data_id(flat_data)
# Validate against empty database == no other datasets with the same external id
validate_external_id_uniqueness(('external_id', ), flat_data, errors,
None)
assert len(errors) == 0
self._create_package_with_unflattened_data(unflattened_data)
# Validate against database containing one dataset that has different external id
validate_external_id_uniqueness(('external_id', ), flat_data, errors,
None)
assert len(errors) == 0
# Validate against database containing one dataset having the same external id
flat_data[('external_id', )] = ext_id_1
self.assertRaises(Invalid, validate_external_id_uniqueness,
('external_id', ), flat_data, errors, None)
def test_validate_primary_pid_uniqueness(self):
errors = defaultdict(list)
unflattened_data = self._get_unflattened_data()
ppid_1 = u'primary_pid_1'
ppid_2 = u'primary_pid_2'
flat_data = flatten_dict(copy.deepcopy(unflattened_data))
unflattened_data['pids'][0]['id'] = ppid_1
flat_data[('pids', 0, 'id')] = ppid_2
self._set_unflattened_data_id(unflattened_data)
self._set_flat_data_id(flat_data)
# Validate against empty database == no other datasets with the same primary pid
validate_primary_pid_uniqueness(('pids', 0, 'id'), flat_data, errors,
None)
assert len(errors) == 0
self._create_package_with_unflattened_data(unflattened_data)
# Validate against database containing one dataset that has different primary pid
validate_primary_pid_uniqueness(('pids', 0, 'id'), flat_data, errors,
None)
assert len(errors) == 0
# Validate against database containing one dataset having the same primary pid
flat_data[('pids', 0, 'id')] = ppid_1
self.assertRaises(Invalid, validate_primary_pid_uniqueness,
('pids', 0, 'id'), flat_data, errors, None)
def flatten_to_string_key(dict):
flattented = flatten_dict(dict)
return untuplize_dict(flattented)