def test_dataset_license(self): indict = { 'licenses': [{ 'name': 'odc-odbl', 'path': 'http://example.com/file.csv', }] } exp = { 'license_id': 'odc-odbl', 'license_title': None, 'license_url': 'http://example.com/file.csv' } out = converter.package(indict) assert out == exp indict = { 'licenses': [{ 'title': 'Open Data Commons Open Database License', 'name': 'odc-odbl' }] } exp = { 'license_id': 'odc-odbl', 'license_title': 'Open Data Commons Open Database License', 'license_url': None } out = converter.package(indict) assert out == exp # Finally, what if more than one license indict = { 'licenses': [{ 'title': 'Open Data Commons Open Database License', 'name': 'odc-pddl' }, { 'title': 'Creative Commons CC Zero License (cc-zero)', 'name': 'cc-zero' }] } exp = { 'license_id': 'odc-pddl', 'license_title': 'Open Data Commons Open Database License', 'license_url': None, 'extras': [{ 'key': 'licenses', 'value': json.dumps(indict['licenses']) }] } out = converter.package(indict) assert out == exp
def test_round_trip_ckan(self): # `ckan1` != `ckan2` but `ckan2` == `ckan3` inpath = 'tests/fixtures/full_ckan_package.json' ckan1 = json.load(open(inpath)) fd1 = ckan_to_frictionless.dataset(ckan1) ckan2 = frictionless_to_ckan.package(fd1) fd2 = ckan_to_frictionless.dataset(ckan2) ckan3 = frictionless_to_ckan.package(fd2) # FIXME: this currently doesn't work for Python 2 due to the way # Unicode is handled and because the dictionary keys do not keep # the same order. # Solution 1: Skip for Python 2 (it's clearly the same dictionary # if the build passes on Python 3) # Solution 2: Hard code the dicts as in `test_extras_is_converted` # in test_frictionless_to_ckan.py instead of loading JSON and # sort the keys. if not six.PY2: assert ckan2 == ckan3
def test_basic_mappings(self): indict = { 'description': 'Country, regional and world GDP in current USD.', 'homepage': 'https://datopian.com' } exp = { 'notes': 'Country, regional and world GDP in current USD.', 'url': 'https://datopian.com' } out = converter.package(indict) assert out == exp
def test_keywords_converted_to_tags(self): keywords = ['economy!!!', 'World Bank'] indict = {'keywords': keywords} out = converter.package(indict) assert out.get('tags') == [ { 'name': 'economy!!!' }, { 'name': 'World Bank' }, ]
def test_passthrough(self): indict = { 'name': 'gdp', 'id': 'xxxx', 'title': 'Countries GDP', 'version': '1.0', } exp = { 'name': 'gdp', 'id': 'xxxx', 'title': 'Countries GDP', 'version': '1.0', } out = converter.package(indict) assert out == exp
def test_passthrough(self): indict = { 'name': 'gdp', 'id': 'xxxx', 'title': 'Countries GDP', 'version': '1.0', "owner_org": "a275814e-6c15-40a8-99fd-af911f1568ef", "metadata_created": "2020-03-31T21:57:48.676558", "metadata_modified": "2020-03-31T21:57:50.215642", "creator_user_id": "b5ab876c-0d04-479a-92de-f66db5dd6fb3", "private": False, "revision_id": "xxx", # TODO: test groups } out = converter.package(indict) assert out == indict
def test_resources_are_converted(self): indict = { 'name': 'gdp', 'resources': [{ 'name': 'data.csv', 'path': 'http://someplace.com/data.csv', 'bytes': 100 }] } exp = { 'name': 'gdp', 'resources': [{ 'name': 'data.csv', 'url': 'http://someplace.com/data.csv', 'size': 100 }] } out = converter.package(indict) assert out == exp
def test_differences_ckan_round_trip(self): # When converting ckan1 to fd1 then fd1 to ckan2, # ckan1 is bound to differ from ckan2. # Those fixtures illustrate the expected differences. inpath = 'tests/fixtures/full_ckan_package.json' ckan1 = json.load(open(inpath)) fd1 = ckan_to_frictionless.dataset(ckan1) ckan2 = frictionless_to_ckan.package(fd1) inpath_round_trip = ('tests/fixtures/' 'full_ckan_package_first_round_trip.json') exp = json.load(open(inpath_round_trip)) # FIXME: this currently doesn't work for Python 2 due to the way # Unicode is handled and because the dictionary keys do not keep # the same order. # Solution 1: Skip for Python 2 (it's clearly the same dictionary # if the build passes on Python 3) # Solution 2: Hard code the dicts as in `test_extras_is_converted` # in test_frictionless_to_ckan.py instead of loading JSON and # sort the keys. if not six.PY2: assert ckan2 == exp
def test_extras_is_converted(self): indict = { 'homepage': 'www.example.com', 'newdict': { 'key1': 'dict_to_jsonify' }, 'newint': 123, 'newkey': 'new value', 'newlist': [1, 2, 3, 'string'], 'title': 'Title here' } exp = { 'title': 'Title here', 'url': 'www.example.com', 'extras': [ { 'key': 'newdict', 'value': '{"key1": "dict_to_jsonify"}' }, { 'key': 'newint', 'value': 123 }, { 'key': 'newkey', 'value': 'new value' }, { 'key': 'newlist', 'value': '[1, 2, 3, "string"]' }, ] } out = converter.package(indict) out['extras'] = sorted(out['extras'], key=lambda i: i['key']) assert out == exp
def test_contributors(self): # author conversion indict = {'contributors': [{'title': 'John Smith'}]} exp = {'author': 'John Smith', 'author_email': None} out = converter.package(indict) assert out == exp # check maintainer conversion indict = { 'contributors': [{ 'title': 'xyz', 'email': '*****@*****.**', 'organisation': 'xxxxx', 'role': 'maintainer' }] } exp = {'maintainer': 'xyz', 'maintainer_email': '*****@*****.**'} out = converter.package(indict) assert out == exp # Make sure that we also get the correct data when there are multiple # contributors indict = { 'contributors': [{ 'title': 'abc', 'email': '*****@*****.**' }, { 'title': 'xyz', 'email': '*****@*****.**', 'role': 'maintainer' }] } exp = { 'author': 'abc', 'author_email': '*****@*****.**', 'maintainer': 'xyz', 'maintainer_email': '*****@*****.**' } out = converter.package(indict) assert out == exp # finally if we have contributors beyond that expected for ckan we keep # that in extras (raw) indict = { 'contributors': [{ "role": "author", "email": "", "title": "Patricio" }, { "role": "maintainer", "email": "", "title": "Rufus" }, { "role": "author", "email": "", "title": "Paul" }] } exp = { 'author': 'Patricio', 'author_email': '', 'maintainer': 'Rufus', 'maintainer_email': '', 'extras': [{ 'key': u'contributors', 'value': json.dumps(indict['contributors']) }] } out = converter.package(indict) assert out == exp
def frictionless_to_dataset(datapackage): """Convert a Frictionless data datapackage dict to a CKAN dataset dict """ return ftc.package(datapackage)