def test_dataset_license(self):
indict = {
'licenses': [{
'name': 'odc-odbl',
'path': 'http://example.com/file.csv',
}]
}
exp = {
'license_id': 'odc-odbl',
'license_title': None,
'license_url': 'http://example.com/file.csv'
}
out = converter.package(indict)
assert out == exp
indict = {
'licenses': [{
'title': 'Open Data Commons Open Database License',
'name': 'odc-odbl'
}]
}
exp = {
'license_id': 'odc-odbl',
'license_title': 'Open Data Commons Open Database License',
'license_url': None
}
out = converter.package(indict)
assert out == exp
# Finally, what if more than one license
indict = {
'licenses': [{
'title': 'Open Data Commons Open Database License',
'name': 'odc-pddl'
}, {
'title': 'Creative Commons CC Zero License (cc-zero)',
'name': 'cc-zero'
}]
}
exp = {
'license_id':
'odc-pddl',
'license_title':
'Open Data Commons Open Database License',
'license_url':
None,
'extras': [{
'key': 'licenses',
'value': json.dumps(indict['licenses'])
}]
}
out = converter.package(indict)
assert out == exp
def test_round_trip_ckan(self):
# `ckan1` != `ckan2` but `ckan2` == `ckan3`
inpath = 'tests/fixtures/full_ckan_package.json'
ckan1 = json.load(open(inpath))
fd1 = ckan_to_frictionless.dataset(ckan1)
ckan2 = frictionless_to_ckan.package(fd1)
fd2 = ckan_to_frictionless.dataset(ckan2)
ckan3 = frictionless_to_ckan.package(fd2)
# FIXME: this currently doesn't work for Python 2 due to the way
# Unicode is handled and because the dictionary keys do not keep
# the same order.
# Solution 1: Skip for Python 2 (it's clearly the same dictionary
# if the build passes on Python 3)
# Solution 2: Hard code the dicts as in `test_extras_is_converted`
# in test_frictionless_to_ckan.py instead of loading JSON and
# sort the keys.
if not six.PY2:
assert ckan2 == ckan3
def test_basic_mappings(self):
indict = {
'description': 'Country, regional and world GDP in current USD.',
'homepage': 'https://datopian.com'
}
exp = {
'notes': 'Country, regional and world GDP in current USD.',
'url': 'https://datopian.com'
}
out = converter.package(indict)
assert out == exp
def test_keywords_converted_to_tags(self):
keywords = ['economy!!!', 'World Bank']
indict = {'keywords': keywords}
out = converter.package(indict)
assert out.get('tags') == [
{
'name': 'economy!!!'
},
{
'name': 'World Bank'
},
]
def test_passthrough(self):
indict = {
'name': 'gdp',
'id': 'xxxx',
'title': 'Countries GDP',
'version': '1.0',
}
exp = {
'name': 'gdp',
'id': 'xxxx',
'title': 'Countries GDP',
'version': '1.0',
}
out = converter.package(indict)
assert out == exp
def test_passthrough(self):
indict = {
'name': 'gdp',
'id': 'xxxx',
'title': 'Countries GDP',
'version': '1.0',
"owner_org": "a275814e-6c15-40a8-99fd-af911f1568ef",
"metadata_created": "2020-03-31T21:57:48.676558",
"metadata_modified": "2020-03-31T21:57:50.215642",
"creator_user_id": "b5ab876c-0d04-479a-92de-f66db5dd6fb3",
"private": False,
"revision_id": "xxx",
# TODO: test groups
}
out = converter.package(indict)
assert out == indict
def test_resources_are_converted(self):
indict = {
'name':
'gdp',
'resources': [{
'name': 'data.csv',
'path': 'http://someplace.com/data.csv',
'bytes': 100
}]
}
exp = {
'name':
'gdp',
'resources': [{
'name': 'data.csv',
'url': 'http://someplace.com/data.csv',
'size': 100
}]
}
out = converter.package(indict)
assert out == exp
def test_differences_ckan_round_trip(self):
# When converting ckan1 to fd1 then fd1 to ckan2,
# ckan1 is bound to differ from ckan2.
# Those fixtures illustrate the expected differences.
inpath = 'tests/fixtures/full_ckan_package.json'
ckan1 = json.load(open(inpath))
fd1 = ckan_to_frictionless.dataset(ckan1)
ckan2 = frictionless_to_ckan.package(fd1)
inpath_round_trip = ('tests/fixtures/'
'full_ckan_package_first_round_trip.json')
exp = json.load(open(inpath_round_trip))
# FIXME: this currently doesn't work for Python 2 due to the way
# Unicode is handled and because the dictionary keys do not keep
# the same order.
# Solution 1: Skip for Python 2 (it's clearly the same dictionary
# if the build passes on Python 3)
# Solution 2: Hard code the dicts as in `test_extras_is_converted`
# in test_frictionless_to_ckan.py instead of loading JSON and
# sort the keys.
if not six.PY2:
assert ckan2 == exp
def test_extras_is_converted(self):
indict = {
'homepage': 'www.example.com',
'newdict': {
'key1': 'dict_to_jsonify'
},
'newint': 123,
'newkey': 'new value',
'newlist': [1, 2, 3, 'string'],
'title': 'Title here'
}
exp = {
'title':
'Title here',
'url':
'www.example.com',
'extras': [
{
'key': 'newdict',
'value': '{"key1": "dict_to_jsonify"}'
},
{
'key': 'newint',
'value': 123
},
{
'key': 'newkey',
'value': 'new value'
},
{
'key': 'newlist',
'value': '[1, 2, 3, "string"]'
},
]
}
out = converter.package(indict)
out['extras'] = sorted(out['extras'], key=lambda i: i['key'])
assert out == exp
def test_contributors(self):
# author conversion
indict = {'contributors': [{'title': 'John Smith'}]}
exp = {'author': 'John Smith', 'author_email': None}
out = converter.package(indict)
assert out == exp
# check maintainer conversion
indict = {
'contributors': [{
'title': 'xyz',
'email': '*****@*****.**',
'organisation': 'xxxxx',
'role': 'maintainer'
}]
}
exp = {'maintainer': 'xyz', 'maintainer_email': '*****@*****.**'}
out = converter.package(indict)
assert out == exp
# Make sure that we also get the correct data when there are multiple
# contributors
indict = {
'contributors': [{
'title': 'abc',
'email': '*****@*****.**'
}, {
'title': 'xyz',
'email': '*****@*****.**',
'role': 'maintainer'
}]
}
exp = {
'author': 'abc',
'author_email': '*****@*****.**',
'maintainer': 'xyz',
'maintainer_email': '*****@*****.**'
}
out = converter.package(indict)
assert out == exp
# finally if we have contributors beyond that expected for ckan we keep
# that in extras (raw)
indict = {
'contributors': [{
"role": "author",
"email": "",
"title": "Patricio"
}, {
"role": "maintainer",
"email": "",
"title": "Rufus"
}, {
"role": "author",
"email": "",
"title": "Paul"
}]
}
exp = {
'author':
'Patricio',
'author_email':
'',
'maintainer':
'Rufus',
'maintainer_email':
'',
'extras': [{
'key': u'contributors',
'value': json.dumps(indict['contributors'])
}]
}
out = converter.package(indict)
assert out == exp
def frictionless_to_dataset(datapackage):
"""Convert a Frictionless data datapackage dict to a CKAN dataset dict
"""
return ftc.package(datapackage)
