def test_dataset_only_requires_a_name_to_be_valid(self):
invalid_dataset_dict = {}
valid_dataset_dict = {
'name': 'gdp',
'resources': [{
'name': 'the-resource',
}]
}
converter.dataset_to_datapackage(valid_dataset_dict)
nose.tools.assert_raises(KeyError, converter.dataset_to_datapackage,
invalid_dataset_dict)
def test_dataset_only_requires_a_name_to_be_valid(self):
invalid_dataset_dict = {}
valid_dataset_dict = {
'name': 'gdp',
'resources': [
{
'name': 'the-resource',
}
]
}
converter.dataset_to_datapackage(valid_dataset_dict)
with self.assertRaises(KeyError):
converter.dataset_to_datapackage(invalid_dataset_dict)
def test_dataset_notes(self):
self.dataset_dict.update({
'notes': 'Country, regional and world GDP in current US Dollars.'
})
result = converter.dataset_to_datapackage(self.dataset_dict)
nose.tools.assert_equals(result.get('description'),
self.dataset_dict['notes'])
def test_dataset_extras(self):
self.dataset_dict.update({
'extras': [
{
'key': 'title_cn',
'value': u'國內生產總值'
},
{
'key': 'years',
'value': '[2015, 2016]'
},
{
'key': 'last_year',
'value': 2016
},
{
'key': 'location',
'value': '{"country": "China"}'
},
]
})
result = converter.dataset_to_datapackage(self.dataset_dict)
nose.tools.assert_equal(
result.get('extras'), {
'title_cn': u'國內生產總值',
'years': [2015, 2016],
'last_year': 2016,
'location': {
'country': 'China'
},
})
def package_show_as_datapackage(context, data_dict):
'''Return the given CKAN dataset into a Data Package.
This returns just the data package metadata in JSON format (what would be
the contents of the datapackage.json file), it does not return the whole
multi-file package including datapackage.json file and additional data
files.
:param id: the ID of the dataset
:type id: string
:returns: the datapackage metadata
:rtype: JSON
'''
try:
dataset_id = data_dict['id']
except KeyError:
raise toolkit.ValidationError({'id': 'missing id'})
dataset_dict = toolkit.get_action('package_show')(context, {
'id': dataset_id
})
return converter.dataset_to_datapackage(dataset_dict)
def test_dataset_notes(self):
self.dataset_dict.update({
'notes': 'Country, regional and world GDP in current US Dollars.'
})
result = converter.dataset_to_datapackage(self.dataset_dict)
self.assertEquals(result.get('description'),
self.dataset_dict['notes'])
def test_dataset_ckan_url(self):
self.dataset_dict.update({
'ckan_url': 'http://www.somewhere.com/datasets/foo'
})
result = converter.dataset_to_datapackage(self.dataset_dict)
self.assertEquals(result.get('homepage'),
self.dataset_dict['ckan_url'])
def test_dataset_ckan_url(self):
self.dataset_dict.update({
'ckan_url': 'http://www.somewhere.com/datasets/foo'
})
result = converter.dataset_to_datapackage(self.dataset_dict)
nose.tools.assert_equals(result.get('homepage'),
self.dataset_dict['ckan_url'])
def test_resource_format(self):
self.resource_dict.update({
'format': 'CSV',
})
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('format'),
self.resource_dict['format'])
def test_resource_url(self):
self.resource_dict.update({
'url': 'http://www.somewhere.com/data.csv',
})
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('path'),
self.resource_dict['url'])
def test_resource_format(self):
self.resource_dict.update({
'format': 'CSV',
})
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('format'),
self.resource_dict['format'])
def test_resource_schema_url(self):
self.resource_dict.update({
'schema': 'http://example.com/some.schema.json'
})
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
self.assertEquals(resource.get('schema'),
self.resource_dict['schema'])
def test_resource_hash(self):
self.resource_dict.update({
'hash': 'e785c0883d7a104330e69aee73d4f235',
})
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('hash'),
self.resource_dict['hash'])
def test_resource_hash(self):
self.resource_dict.update({
'hash': 'e785c0883d7a104330e69aee73d4f235',
})
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('hash'),
self.resource_dict['hash'])
def test_resource_description(self):
self.resource_dict.update({
'description': 'GDPs list',
})
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('description'),
self.resource_dict['description'])
def test_dataset_maintainer(self):
author = {'name': 'John Smith', 'email': '*****@*****.**'}
self.dataset_dict.update({
'maintainer': author['name'],
'maintainer_email': author['email'],
})
result = converter.dataset_to_datapackage(self.dataset_dict)
nose.tools.assert_equals(result.get('author'), author)
def test_resource_url(self):
self.resource_dict.update({
'url': 'http://www.somewhere.com/data.csv',
})
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('path'),
self.resource_dict['url'])
def test_resource_description(self):
self.resource_dict.update({
'description': 'GDPs list',
})
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('description'),
self.resource_dict['description'])
def test_dataset_only_requires_a_name_to_be_valid(self):
invalid_dataset_dict = {}
valid_dataset_dict = {
'name': 'gdp',
'resources': [
{
'name': 'the-resource',
}
]
}
converter.dataset_to_datapackage(valid_dataset_dict)
nose.tools.assert_raises(
KeyError,
converter.dataset_to_datapackage,
invalid_dataset_dict
)
def test_resource_name_slugifies_the_name(self):
self.resource_dict.update({
'name': 'Lista de PIBs dos países! 51',
})
expected_name = 'lista-de-pibs-dos-paises-51'
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('name'), expected_name)
nose.tools.assert_equals(resource.get('title'),
self.resource_dict['name'])
def test_dataset_name_title_and_version(self):
self.dataset_dict.update({
'name': 'gdp',
'title': 'Countries GDP',
'version': '1.0',
})
result = converter.dataset_to_datapackage(self.dataset_dict)
self.assertEquals(result['title'], self.dataset_dict['title'])
self.assertEquals(result['name'], self.dataset_dict['name'])
self.assertEquals(result['version'], self.dataset_dict['version'])
def test_resource_path_is_set_even_for_uploaded_resources(self):
self.resource_dict.update({
'id': 'foo',
'url': 'http://www.somewhere.com/data.csv',
'url_type': 'upload',
})
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('path'),
self.resource_dict['url'])
def test_resource_name_converts_unicode_characters(self):
self.resource_dict.update({
'name': u'万事开头难',
})
expected_name = 'mo-shi-kai-tou-nan'
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('name'), expected_name)
nose.tools.assert_equals(resource.get('title'),
self.resource_dict['name'])
def test_resource_name_lowercases_the_name(self):
self.resource_dict.update({
'name': 'ThE-nAmE',
})
expected_name = 'the-name'
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('name'), expected_name)
nose.tools.assert_equals(resource.get('title'),
self.resource_dict['name'])
def test_dataset_name_title_and_version(self):
self.dataset_dict.update({
'name': 'gdp',
'title': 'Countries GDP',
'version': '1.0',
})
result = converter.dataset_to_datapackage(self.dataset_dict)
nose.tools.assert_equals(result['title'], self.dataset_dict['title'])
nose.tools.assert_equals(result['name'], self.dataset_dict['name'])
nose.tools.assert_equals(result['version'], self.dataset_dict['version'])
def test_resource_path_is_set_even_for_uploaded_resources(self):
self.resource_dict.update({
'id': 'foo',
'url': 'http://www.somewhere.com/data.csv',
'url_type': 'upload',
})
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('path'),
self.resource_dict['url'])
def test_resource_name_lowercases_the_name(self):
self.resource_dict.update({
'name': 'ThE-nAmE',
})
expected_name = 'the-name'
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('name'),
expected_name)
nose.tools.assert_equals(resource.get('title'),
self.resource_dict['name'])
def test_dataset_maintainer(self):
author = {
'name': 'John Smith',
'email': '*****@*****.**'
}
self.dataset_dict.update({
'maintainer': author['name'],
'maintainer_email': author['email'],
})
result = converter.dataset_to_datapackage(self.dataset_dict)
nose.tools.assert_equals(result.get('author'), author)
def test_resource_name_slugifies_the_name(self):
self.resource_dict.update({
'name': 'Lista de PIBs dos países! 51',
})
expected_name = 'lista-de-pibs-dos-paises-51'
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('name'),
expected_name)
nose.tools.assert_equals(resource.get('title'),
self.resource_dict['name'])
def test_resource_name_converts_unicode_characters(self):
self.resource_dict.update({
'name': u'万事开头难',
})
expected_name = 'mo-shi-kai-tou-nan'
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
nose.tools.assert_equals(resource.get('name'),
expected_name)
nose.tools.assert_equals(resource.get('title'),
self.resource_dict['name'])
def test_resource_schema(self):
self.resource_dict.update({
'schema': {
'fields': [
{'name': 'id', 'type': 'integer'},
{'name': 'title', 'type': 'string'},
]
}
})
result = converter.dataset_to_datapackage(self.dataset_dict)
resource = result.get('resources')[0]
self.assertEquals(resource.get('schema'),
self.resource_dict['schema'])
def test_dataset_license(self):
license = {
'type': 'cc-zero',
'title': 'Creative Commons CC Zero License (cc-zero)',
'url': 'http://opendefinition.org/licenses/cc-zero/'
}
self.dataset_dict.update({
'license_id': license['type'],
'license_title': license['title'],
'license_url': license['url'],
})
result = converter.dataset_to_datapackage(self.dataset_dict)
nose.tools.assert_equals(result.get('license'), license)
def test_dataset_license(self):
license = {
'type': 'cc-zero',
'title': 'Creative Commons CC Zero License (cc-zero)',
'url': 'http://opendefinition.org/licenses/cc-zero/'
}
self.dataset_dict.update({
'license_id': license['type'],
'license_title': license['title'],
'license_url': license['url'],
})
result = converter.dataset_to_datapackage(self.dataset_dict)
nose.tools.assert_equals(result.get('license'), license)
def test_dataset_author_and_source(self):
sources = [{
'name': 'World Bank and OECD',
'email': '*****@*****.**',
'web': 'http://data.worldbank.org/indicator/NY.GDP.MKTP.CD',
}]
self.dataset_dict.update({
'author': sources[0]['name'],
'author_email': sources[0]['email'],
'url': sources[0]['web']
})
result = converter.dataset_to_datapackage(self.dataset_dict)
nose.tools.assert_equals(result.get('sources'), sources)
def test_dataset_author_and_source(self):
sources = [
{
'name': 'World Bank and OECD',
'email': '*****@*****.**',
'web': 'http://data.worldbank.org/indicator/NY.GDP.MKTP.CD',
}
]
self.dataset_dict.update({
'author': sources[0]['name'],
'author_email': sources[0]['email'],
'url': sources[0]['web']
})
result = converter.dataset_to_datapackage(self.dataset_dict)
nose.tools.assert_equals(result.get('sources'), sources)
def test_dataset_extras(self):
self.dataset_dict.update({
'extras': [
{'key': 'title_cn', 'value': u'國內生產總值'},
{'key': 'years', 'value': '[2015, 2016]'},
{'key': 'last_year', 'value': 2016},
{'key': 'location', 'value': '{"country": "China"}'},
]
})
result = converter.dataset_to_datapackage(self.dataset_dict)
nose.tools.assert_equal(result.get('extras'), {
'title_cn': u'國內生產總值',
'years': [2015, 2016],
'last_year': 2016,
'location': {'country': 'China'},
})
def test_package_show_as_datapackage(self):
dataset = factories.Dataset()
factories.Resource(package_id=dataset['id'], url='http://test.com/test-url-1',
schema='{"fields":[{"type":"string", "name":"col1"}]}')
factories.Resource(package_id=dataset['id'], url='http://test.com/test-url-2',
schema='{"fields":[{"type":"string", "name":"col1"}]}')
expected_output = converter.dataset_to_datapackage(
helpers.call_action('package_show', id=dataset['id'])
)
datapackage_dict = helpers.call_action('package_show_as_datapackage',
id=dataset['name'])
nose.tools.assert_items_equal(expected_output, datapackage_dict)
def test_dataset_tags(self):
keywords = ['economy', 'worldbank']
self.dataset_dict.update({
'tags': [{
'display_name': 'economy',
'id': '9d602a79-7742-44a7-9029-50b9eca38c90',
'name': 'economy',
'state': 'active'
}, {
'display_name': 'worldbank',
'id': '3ccc2e3b-f875-49ef-a39d-6601d6c0ef76',
'name': 'worldbank',
'state': 'active'
}]
})
result = converter.dataset_to_datapackage(self.dataset_dict)
nose.tools.assert_equals(result.get('keywords'), keywords)
def test_package_show_as_datapackage(self):
dataset = factories.Dataset()
factories.Resource(
package_id=dataset['id'],
url='http://test.com/test-url-1',
schema='{"fields":[{"type":"string", "name":"col1"}]}')
factories.Resource(
package_id=dataset['id'],
url='http://test.com/test-url-2',
schema='{"fields":[{"type":"string", "name":"col1"}]}')
expected_output = converter.dataset_to_datapackage(
helpers.call_action('package_show', id=dataset['id']))
datapackage_dict = helpers.call_action('package_show_as_datapackage',
id=dataset['name'])
nose.tools.assert_items_equal(expected_output, datapackage_dict)
def test_dataset_tags(self):
keywords = [
'economy', 'worldbank'
]
self.dataset_dict.update({
'tags': [
{
'display_name': 'economy',
'id': '9d602a79-7742-44a7-9029-50b9eca38c90',
'name': 'economy',
'state': 'active'
},
{
'display_name': 'worldbank',
'id': '3ccc2e3b-f875-49ef-a39d-6601d6c0ef76',
'name': 'worldbank',
'state': 'active'
}
]
})
result = converter.dataset_to_datapackage(self.dataset_dict)
nose.tools.assert_equals(result.get('keywords'), keywords)
def package_show_as_datapackage(context, data_dict):
'''Return the given CKAN dataset into a Data Package.
This returns just the data package metadata in JSON format (what would be
the contents of the datapackage.json file), it does not return the whole
multi-file package including datapackage.json file and additional data
files.
:param id: the ID of the dataset
:type id: string
:returns: the datapackage metadata
:rtype: JSON
'''
try:
dataset_id = data_dict['id']
except KeyError:
raise toolkit.ValidationError({'id': 'missing id'})
dataset_dict = toolkit.get_action('package_show')(context,
{'id': dataset_id})
return converter.dataset_to_datapackage(dataset_dict)
def test_basic_dataset_in_setup_is_valid(self):
converter.dataset_to_datapackage(self.dataset_dict)
def test_basic_dataset_in_setup_is_valid(self):
converter.dataset_to_datapackage(self.dataset_dict)
