def populate_theme_groups(instance, clean_existing=False):
"""
For given instance, it finds groups from mapping corresponding to
Dataset's themes, and will assign dataset to those groups.
Existing groups will be removed, if clean_existing is set to True.
This utilizes `ckanext.dcatapit.theme_group_mapping.add_new_groups`
configuration option. If it's set to true, and mapped group doesn't exist,
new group will be created.
"""
add_new = toolkit.asbool(
config.get(DCATAPIT_THEME_TO_MAPPING_ADD_NEW_GROUPS))
themes = []
for ex in (instance.get('extras') or []):
if ex['key'] == 'theme':
_t = ex['value']
if isinstance(_t, list):
themes.extend(_t)
else:
try:
tval = json.loads(_t)
except Exception:
tval = [{
'theme': t,
'subthemes': []
} for t in _decode_list(_t)]
for tv in tval:
themes.append(tv['theme'])
if not themes:
log.debug("no theme from %s", instance)
return instance
theme_map = get_theme_to_groups()
if not theme_map:
log.warning("Theme to group map is empty")
return instance
if not isinstance(themes, list):
themes = [themes]
all_groups = set()
for theme in themes:
_groups = theme_map.get(theme)
if not _groups:
continue
all_groups = all_groups.union(set(_groups))
if clean_existing:
_clean_groups(instance)
groups = []
for gname in all_groups:
gname = gname.strip()
if not gname:
continue
group = Group.get(gname) or _get_group_from_session(gname)
if add_new and group is None:
group = Group(name=gname)
Session.add(group)
if group:
groups.append(group)
if Session.new:
# flush to db, refresh with ids
rev = Session.revision
Session.flush()
Session.revision = rev
groups = [(Group.get(g.name) if g.id is None else g) for g in groups]
_add_groups(instance['id'], set(groups))
# preserve revision, since it's not a commit yet
rev = Session.revision
Session.flush()
Session.revision = rev
return instance
def populate_theme_groups(instance, clean_existing=False):
"""
For given instance, it finds groups from mapping corresponding to
Dataset's themes, and will assign dataset to those groups.
Existing groups will be removed, if clean_existing is set to True.
This utilizes `ckanext.dcatapit.theme_group_mapping.add_new_groups`
configuration option. If it's set to true, and mapped group doesn't exist,
new group will be created.
"""
add_new = toolkit.asbool(
config.get(DCATAPIT_THEME_TO_MAPPING_ADD_NEW_GROUPS))
themes = []
for ex in (instance.get('extras') or []):
if ex['key'] == FIELD_THEMES_AGGREGATE:
_t = ex['value']
if isinstance(_t, list):
themes.extend(_t)
else:
try:
tval = json.loads(_t)
except Exception:
log.warning(f'Trying old themes format for {_t}')
tval = [{
'theme': t,
'subthemes': []
} for t in _t.strip('{}').split(',')]
for tv in tval:
themes.append(tv['theme'])
break # we don't need any other info - if there are 'themes' is ok to bypass them
elif ex['key'] == 'theme':
_t = ex['value']
if isinstance(_t, list):
themes.extend(_t)
else:
try:
tval = json.loads(_t)
except Exception:
log.warning(f'Trying old themes format for {_t}')
tval = _t.strip('{}').split(',')
themes.extend(tval)
# dont break the for loop: if aggregates are there, they get precedence
if not themes:
log.debug('no theme from %s', instance)
return instance
theme_map = get_theme_to_groups()
if not theme_map:
log.warning('Theme to group map is empty')
return instance
if not isinstance(themes, list):
themes = [themes]
all_groups = set()
for theme in themes:
_groups = theme_map.get(theme)
if not _groups:
continue
all_groups = all_groups.union(set(_groups))
if clean_existing:
_clean_groups(instance)
groups = []
for gname in all_groups:
gname = gname.strip()
if not gname:
continue
group = Group.get(gname) or _get_group_from_session(gname)
if add_new and group is None:
group = Group(name=gname)
Session.add(group)
if group:
groups.append(group)
if Session.new:
# flush to db, refresh with ids
Session.flush()
groups = [(Group.get(g.name) if g.id is None else g) for g in groups]
_add_groups(instance['id'], set(groups))
Session.flush()
return instance
