def test_ckan_harvester_license(self):
dataset = {
'title':
'some title',
'id':
'sometitle',
'resources': [{
'id': 'resource/1111',
'url': 'http://resource/1111',
'license_type': 'invalid',
}, {
'id':
'resource/2222',
'url':
'http://resource/2222',
'license_type':
'https://w3id.org/italia/controlled-vocabulary/licences/A311_GFDL13'
}]
}
data = json.dumps(dataset)
harvest_dict = self._create_harvest_obj('http://mock/source/',
name='testpkg')
harvest_obj = HarvestObject.get(harvest_dict['id'])
harvest_obj.content = data
h = CKANMappingHarvester()
h.import_stage(harvest_obj)
Session.flush()
pkg_dict = helpers.call_action('package_show',
context={},
name_or_id='sometitle')
self.assertTrue(len(pkg_dict['resources']) == 2)
resources = pkg_dict['resources']
r = dataset['resources']
for res in resources:
if res['id'] == r[0]['id']:
self.assertEqual(res['license_type'],
License.get(License.DEFAULT_LICENSE).uri)
else:
self.assertEqual(res['license_type'], r[1]['license_type'])
def test_groups_to_themes_mapping(self):
config[DCATAPIT_THEMES_MAP] = os.path.join(os.path.dirname(__file__),
'..', '..', '..',
'examples',
'themes_mapping.json')
url = 'http://some.test.harvest.url'
groups_non_mappable = [{
'name': 'non-mappable',
'display_name': 'non-mappable'
}], []
groups_mappable = [{'name': 'agriculture', 'display_name': 'agricoltura-e-allevamento', 'identifier': 'dummy'}],\
[{'key': 'theme', 'value': 'AGRI'}]
mapped_theme_name = 'AGRI'
harvest_obj = self._make_harvest_object(url, groups_non_mappable[0])
harvester = CKANMappingHarvester()
licenses_file = get_voc_file(LICENSES_FILE)
self.g = load_graph(path=licenses_file)
load_licenses(self.g)
Session.flush()
# clean, no mapping
harvester.import_stage(harvest_obj)
hdata = json.loads(harvest_obj.content)
self.assertEqual(
[t for t in hdata.get('extras', []) if t['key'] == 'theme'], [])
# test mapping
hdata = json.loads(harvest_obj.content)
hdata['groups'] = groups_mappable[0]
harvest_obj.content = json.dumps(hdata)
harvester.import_stage(harvest_obj)
hdata = json.loads(harvest_obj.content)
extras = hdata.get('extras', [])
# self.assertEqual([t for t in hdata.get('extras', []) if t['key'] == 'theme'], groups_mappable[1])
theme_uri_list_extra = _get_extra(extras, 'theme')
self.assertIsNotNone(theme_uri_list_extra)
theme_uri_list = json.loads(theme_uri_list_extra['value'])
expected_theme_uri_list = [theme_name_to_uri(mapped_theme_name)]
self.assertListEqual(expected_theme_uri_list, theme_uri_list)
aggr_extra = _get_extra(extras, FIELD_THEMES_AGGREGATE)
self.assertIsNotNone(aggr_extra)
aggr = json.loads(aggr_extra['value'])
self.assertEqual(mapped_theme_name, aggr[0]['theme'])
def test_groups_to_themes_mapping(self):
config[DCATAPIT_THEMES_MAP] = os.path.join(os.path.dirname(__file__),
'..',
'..',
'..',
'examples',
'themes_mapping.json')
url = 'http://some.test.harvest.url'
groups_non_mappable = [{'name': 'non-mappable', 'display_name': 'non-mappable'}], []
groups_mappable = [{'name': 'agriculture', 'display_name': 'agricoltura-e-allevamento'}],\
[{'key': 'theme', 'value': 'AGRI'}]
harvest_obj = self._make_harvest_object(url, groups_non_mappable[0])
harvester = CKANMappingHarvester()
def get_path(fname):
return os.path.join(os.path.dirname(__file__),
'..', '..', '..', 'examples', fname)
licenses = get_path('licenses.rdf')
self.g = _get_graph(path=licenses)
load_from_graph(path=licenses)
rev = getattr(Session, 'revision', None)
Session.flush()
Session.revision = rev
# clean, no mapping
harvester.import_stage(harvest_obj)
hdata = json.loads(harvest_obj.content)
eq_([t for t in hdata.get('extras', []) if t['key'] == 'theme'], [])
# test mapping
hdata = json.loads(harvest_obj.content)
hdata['groups'] = groups_mappable[0]
harvest_obj.content = json.dumps(hdata)
harvester.import_stage(harvest_obj)
hdata = json.loads(harvest_obj.content)
eq_([t for t in hdata.get('extras', []) if t['key'] == 'theme'], groups_mappable[1])