def test_mapping(self):
# multilang requires lang to be set
from pylons.i18n.translation import set_lang, get_lang
import pylons
class dummyreq(object):
class p(object):
translator = object()
environ = {'pylons.pylons': p()}
pylons.request = dummyreq()
pylons.translator.pylons_lang = ['en_GB']
set_lang('en_GB')
assert get_lang() == ['en_GB']
assert 'dcatapit_theme_group_mapper' in config['ckan.plugins'], "No dcatapit_theme_group_mapper plugin in config"
contents = self._get_file_contents('dataset.rdf')
p = RDFParser(profiles=['it_dcat_ap'])
p.parse(contents)
datasets = [d for d in p.datasets()]
eq_(len(datasets), 1)
package_dict = datasets[0]
user = User.get('dummy')
if not user:
user = call_action('user_create',
name='dummy',
password='******',
email='*****@*****.**')
user_name = user['name']
else:
user_name = user.name
org = Group.by_name('dummy')
if org is None:
org = call_action('organization_create',
context={'user': user_name},
name='dummy',
identifier='aaaaaa')
existing_g = Group.by_name('existing-group')
if existing_g is None:
existing_g = call_action('group_create',
context={'user': user_name},
name='existing-group')
context = {'user': '******',
'ignore_auth': True,
'defer_commit': False}
package_schema = schema.default_create_package_schema()
context['schema'] = package_schema
_p = {'frequency': 'manual',
'publisher_name': 'dummy',
'extras': [{'key':'theme', 'value':['non-mappable', 'thememap1']}],
'groups': [],
'title': 'dummy',
'holder_name': 'dummy',
'holder_identifier': 'dummy',
'name': 'dummy',
'notes': 'dummy',
'owner_org': 'dummy',
'modified': datetime.now(),
'publisher_identifier': 'dummy',
'metadata_created' : datetime.now(),
'metadata_modified': datetime.now(),
'guid': unicode(uuid.uuid4),
'identifier': 'dummy'}
package_dict.update(_p)
config[DCATAPIT_THEME_TO_MAPPING_SOURCE] = ''
package_data = call_action('package_create', context=context, **package_dict)
p = Package.get(package_data['id'])
# no groups should be assigned at this point (no map applied)
assert {'theme': ['non-mappable', 'thememap1']} == p.extras, '{} vs {}'.format(_p['extras'], p.extras)
assert [] == p.get_groups(group_type='group'), 'should be {}, got {}'.format([], p.get_groups(group_type='group'))
package_data = call_action('package_show', context=context, id=package_data['id'])
# use test mapping, which replaces thememap1 to thememap2 and thememap3
test_map_file = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'examples', 'test_map.ini')
config[DCATAPIT_THEME_TO_MAPPING_SOURCE] = test_map_file
package_dict['theme'] = ['non-mappable', 'thememap1']
expected_groups_existing = ['existing-group']
expected_groups_new = expected_groups_existing + ['somegroup1', 'somegroup2']
expected_groups_multi = expected_groups_new + ['othergroup']
package_dict.pop('extras', None)
p = Package.get(package_data['id'])
context['package'] = p
package_data = call_action('package_update',
context=context,
**package_dict)
#meta.Session.flush()
#meta.Session.revision = repo.new_revision()
# check - only existing group should be assigned
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
assert expected_groups_existing == groups, (expected_groups_existing, 'vs', groups,)
config[DCATAPIT_THEME_TO_MAPPING_ADD_NEW_GROUPS] = 'true'
package_dict['theme'] = ['non-mappable', 'thememap1']
package_data = call_action('package_update', context=context, **package_dict)
meta.Session.flush()
meta.Session.revision = repo.new_revision()
# recheck - this time, new groups should appear
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
assert len(expected_groups_new) == len(groups), (expected_groups_new, 'vs', groups,)
assert set(expected_groups_new) == set(groups), (expected_groups_new, 'vs', groups,)
package_dict['theme'] = ['non-mappable', 'thememap1', 'thememap-multi']
package_data = call_action('package_update', context=context, **package_dict)
meta.Session.flush()
meta.Session.revision = repo.new_revision()
# recheck - there should be no duplicates
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
assert len(expected_groups_multi) == len(groups), (expected_groups_multi, 'vs', groups,)
assert set(expected_groups_multi) == set(groups), (expected_groups_multi, 'vs', groups,)
package_data = call_action('package_update', context=context, **package_dict)
meta.Session.flush()
meta.Session.revision = repo.new_revision()
# recheck - there still should be no duplicates
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
assert len(expected_groups_multi) == len(groups), (expected_groups_multi, 'vs', groups,)
assert set(expected_groups_multi) == set(groups), (expected_groups_multi, 'vs', groups,)
meta.Session.rollback()
def test_theme_to_group_mapping(self):
# multilang requires lang to be set
# class dummyreq(object):
# class p(object):
# translator = object()
# environ = {'pylons.pylons': p()}
# CKANRequest(dummyreq)
# pylons.request = dummyreq()
# pylons.translator.pylons_lang = ['en_GB']
#set_lang('en_GB')
#assert get_lang() == ['en_GB']
assert 'dcatapit_theme_group_mapper' in config[
'ckan.plugins'], 'No dcatapit_theme_group_mapper plugin in config'
with open(get_example_file('dataset.rdf'), 'r') as f:
contents = f.read()
p = RDFParser(profiles=['it_dcat_ap'])
p.parse(contents)
datasets = [d for d in p.datasets()]
self.assertEqual(len(datasets), 1)
package_dict = datasets[0]
user = User.get('dummy')
if not user:
user = call_action('user_create',
name='dummy',
password='******',
email='*****@*****.**')
user_name = user['name']
else:
user_name = user.name
org = Group.by_name('dummy')
if org is None:
org = call_action('organization_create',
context={'user': user_name},
name='dummy',
identifier='aaaaaa')
existing_g = Group.by_name('existing-group')
if existing_g is None:
existing_g = call_action('group_create',
context={'user': user_name},
name='existing-group')
context = {'user': '******', 'ignore_auth': True, 'defer_commit': False}
package_schema = schema.default_create_package_schema()
context['schema'] = package_schema
_p = {
'frequency': 'manual',
'publisher_name': 'dummy',
'extras': [{
'key': 'theme',
'value': ['non-mappable', 'thememap1']
}],
'groups': [], # [{'name':existing_g.name}],
'title': 'dummy',
'holder_name': 'dummy',
'holder_identifier': 'dummy',
'name': 'dummy-' + uuid4().hex,
'identifier': 'dummy' + uuid4().hex,
'notes': 'dummy',
'owner_org': 'dummy',
'modified': datetime.now(),
'publisher_identifier': 'dummy',
'metadata_created': datetime.now(),
'metadata_modified': datetime.now(),
'guid': str(uuid.uuid4),
}
package_dict.update(_p)
config[DCATAPIT_THEME_TO_MAPPING_SOURCE] = ''
config[DCATAPIT_THEME_TO_MAPPING_ADD_NEW_GROUPS] = 'false'
package_data = call_action('package_create',
context=context,
**package_dict)
p = Package.get(package_data['id'])
# no groups should be assigned at this point (no map applied)
assert {
'theme': ['non-mappable', 'thememap1']
} == p.extras, '{} vs {}'.format(_p['extras'], p.extras)
assert [] == p.get_groups(
group_type='group'), 'should be {}, got {}'.format(
[], p.get_groups(group_type='group'))
package_data = call_action('package_show',
context=context,
id=package_data['id'])
# use test mapping, which replaces thememap1 to thememap2 and thememap3
test_map_file = os.path.join(os.path.dirname(__file__), '..', '..',
'..', 'examples', 'test_map.ini')
config[DCATAPIT_THEME_TO_MAPPING_SOURCE] = test_map_file
config[DCATAPIT_THEME_TO_MAPPING_ADD_NEW_GROUPS] = 'false'
# package_dict['theme'] = ['non-mappable', 'thememap1']
package_dict.pop('extras', None)
p = Package.get(package_data['id'])
context['package'] = p
package_data = call_action('package_update',
context=context,
**package_dict)
# check - only existing group should be assigned
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
# the map file maps ECON to existing group, and 2 other unexisting groups that will not be created
expected_groups = ['existing-group']
self.assertSetEqual(set(expected_groups), set(groups),
'Error in assigned groups')
config[DCATAPIT_THEME_TO_MAPPING_SOURCE] = test_map_file
config[DCATAPIT_THEME_TO_MAPPING_ADD_NEW_GROUPS] = 'true'
# package_dict['theme'] = ['non-mappable', 'thememap1']
package_data = call_action('package_update',
context=context,
**package_dict)
meta.Session.flush()
# recheck - this time, new groups should appear
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
# the map file maps ECON to existing group and 2 other groups that have been automatically created
expected_groups = expected_groups + ['somegroup1', 'somegroup2']
self.assertSetEqual(set(expected_groups), set(groups), 'Groups differ')
# package_dict['theme'] = ['non-mappable', 'thememap1', 'thememap-multi']
aggr = json.loads(package_dict[FIELD_THEMES_AGGREGATE])
aggr.append({'theme': 'thememap-multi', 'subthemes': []})
package_dict[FIELD_THEMES_AGGREGATE] = json.dumps(aggr)
package_data = call_action('package_update',
context=context,
**package_dict)
meta.Session.flush()
# recheck - there should be no duplicates
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
# added theme 'thememap-multi', that maps to 'othergroup' and other already exisintg groups
expected_groups = expected_groups + ['othergroup']
self.assertEqual(len(expected_groups), len(groups),
'New groups differ - there may be duplicated groups')
self.assertSetEqual(set(expected_groups), set(groups),
'New groups differ')
package_data = call_action('package_update',
context=context,
**package_dict)
meta.Session.flush()
# recheck - there still should be no duplicates
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
self.assertEqual(len(expected_groups), len(groups),
'New groups differ - there may be duplicated groups')
self.assertSetEqual(set(expected_groups), set(groups),
'New groups differ')
meta.Session.rollback()
