def create_arbitrary(cls, package_dicts, relationships=[],
extra_user_names=[], extra_group_names=[],
admins=[]):
'''Creates packages and a few extra objects as well at the
same time if required.
@param package_dicts - a list of dictionaries with the package
properties.
Extra keys allowed:
"admins" - list of user names to make admin
for this package.
@param extra_group_names - a list of group names to create. No
properties get set though.
@param admins - a list of user names to make admins of all the
packages created.
'''
assert isinstance(relationships, (list, tuple))
assert isinstance(extra_user_names, (list, tuple))
assert isinstance(extra_group_names, (list, tuple))
model.Session.remove()
new_user_names = extra_user_names
new_group_names = set()
new_groups = {}
rev = model.repo.new_revision()
rev.author = cls.author
rev.message = u'Creating test packages.'
admins_list = defaultdict(list) # package_name: admin_names
if package_dicts:
if isinstance(package_dicts, dict):
package_dicts = [package_dicts]
for item in package_dicts:
pkg_dict = {}
for field in cls.pkg_core_fields:
if item.has_key(field):
pkg_dict[field] = unicode(item[field])
if model.Package.by_name(pkg_dict['name']):
log.warning('Cannot create package "%s" as it already exists.' % \
(pkg_dict['name']))
continue
pkg = model.Package(**pkg_dict)
model.Session.add(pkg)
for attr, val in item.items():
if isinstance(val, str):
val = unicode(val)
if attr=='name':
continue
if attr in cls.pkg_core_fields:
pass
elif attr == 'download_url':
pkg.add_resource(unicode(val))
elif attr == 'resources':
assert isinstance(val, (list, tuple))
for res_dict in val:
non_extras = {}
for k, v in res_dict.items():
if k != 'extras':
if not isinstance(v, datetime.datetime):
v = unicode(v)
non_extras[str(k)] = v
extras = dict([(str(k), unicode(v)) for k, v in res_dict.get('extras', {}).items()])
pkg.add_resource(extras=extras, **non_extras)
elif attr == 'tags':
if isinstance(val, (str, unicode)):
tags = val.split()
elif isinstance(val, list):
tags = val
else:
raise NotImplementedError
for tag_name in tags:
tag_name = unicode(tag_name)
tag = model.Tag.by_name(tag_name)
if not tag:
tag = model.Tag(name=tag_name)
cls.tag_names.append(tag_name)
model.Session.add(tag)
pkg.add_tag(tag)
model.Session.flush()
elif attr == 'groups':
model.Session.flush()
if isinstance(val, (str, unicode)):
group_names = val.split()
elif isinstance(val, list):
group_names = val
else:
raise NotImplementedError
for group_name in group_names:
group = model.Group.by_name(unicode(group_name))
if not group:
if not group_name in new_groups:
group = model.Group(name=unicode(group_name))
model.Session.add(group)
new_group_names.add(group_name)
new_groups[group_name] = group
else:
# If adding multiple packages with the same group name,
# model.Group.by_name will not find the group as the
# session has not yet been committed at this point.
# Fetch from the new_groups dict instead.
group = new_groups[group_name]
member = model.Member(group=group, table_id=pkg.id, table_name='package')
model.Session.add(member)
elif attr == 'license':
pkg.license_id = val
elif attr == 'license_id':
pkg.license_id = val
elif attr == 'extras':
pkg.extras = val
elif attr == 'admins':
assert isinstance(val, list)
admins_list[item['name']].extend(val)
for user_name in val:
if user_name not in new_user_names:
new_user_names.append(user_name)
else:
raise NotImplementedError(attr)
cls.pkg_names.append(item['name'])
model.setup_default_user_roles(pkg, admins=[])
for admin in admins:
admins_list[item['name']].append(admin)
model.repo.commit_and_remove()
needs_commit = False
rev = model.repo.new_revision()
for group_name in extra_group_names:
group = model.Group(name=unicode(group_name))
model.Session.add(group)
new_group_names.add(group_name)
needs_commit = True
if needs_commit:
model.repo.commit_and_remove()
needs_commit = False
# create users that have been identified as being needed
for user_name in new_user_names:
if not model.User.by_name(unicode(user_name)):
user = model.User(name=unicode(user_name))
model.Session.add(user)
cls.user_refs.append(user_name)
needs_commit = True
if needs_commit:
model.repo.commit_and_remove()
needs_commit = False
# setup authz for admins
for pkg_name, admins in admins_list.items():
pkg = model.Package.by_name(unicode(pkg_name))
admins_obj_list = []
for admin in admins:
if isinstance(admin, model.User):
admin_obj = admin
else:
admin_obj = model.User.by_name(unicode(admin))
assert admin_obj, admin
admins_obj_list.append(admin_obj)
model.setup_default_user_roles(pkg, admins_obj_list)
needs_commit = True
# setup authz for groups just created
for group_name in new_group_names:
group = model.Group.by_name(unicode(group_name))
model.setup_default_user_roles(group)
cls.group_names.add(group_name)
needs_commit = True
if needs_commit:
model.repo.commit_and_remove()
needs_commit = False
if relationships:
rev = model.repo.new_revision()
rev.author = cls.author
rev.message = u'Creating package relationships.'
def pkg(pkg_name):
return model.Package.by_name(unicode(pkg_name))
for subject_name, relationship, object_name in relationships:
pkg(subject_name).add_relationship(
unicode(relationship), pkg(object_name))
needs_commit = True
model.repo.commit_and_remove()
def create(cls, auth_profile="", package_type=None):
model.Session.remove()
rev = model.repo.new_revision()
# same name as user we create below
rev.author = cls.author
rev.message = u'''Creating test data.
* Package: annakarenina
* Package: warandpeace
* Associated tags, etc etc
'''
if auth_profile == "publisher":
organization_group = model.Group(name=u"organization_group", type="organization")
cls.pkg_names = [u'annakarenina', u'warandpeace']
pkg1 = model.Package(name=cls.pkg_names[0], type=package_type)
if auth_profile == "publisher":
pkg1.group = organization_group
model.Session.add(pkg1)
pkg1.title = u'A Novel By Tolstoy'
pkg1.version = u'0.7a'
pkg1.url = u'http://www.annakarenina.com'
# put an & in the url string to test escaping
if 'alt_url' in model.Resource.get_extra_columns():
configured_extras = ({'alt_url': u'alt123'},
{'alt_url': u'alt345'})
else:
configured_extras = ({}, {})
pr1 = model.Resource(
url=u'http://www.annakarenina.com/download/x=1&y=2',
format=u'plain text',
description=u'Full text. Needs escaping: " Umlaut: \xfc',
hash=u'abc123',
extras={'size_extra': u'123'},
**configured_extras[0]
)
pr2 = model.Resource(
url=u'http://www.annakarenina.com/index.json',
format=u'json',
description=u'Index of the novel',
hash=u'def456',
extras={'size_extra': u'345'},
**configured_extras[1]
)
model.Session.add(pr1)
model.Session.add(pr2)
pkg1.resource_groups_all[0].resources_all.append(pr1)
pkg1.resource_groups_all[0].resources_all.append(pr2)
pkg1.notes = u'''Some test notes
### A 3rd level heading
**Some bolded text.**
*Some italicized text.*
Foreign characters:
u with umlaut \xfc
66-style quote \u201c
foreign word: th\xfcmb
Needs escaping:
left arrow <
<http://ckan.net/>
'''
pkg2 = model.Package(name=cls.pkg_names[1], type=package_type)
tag1 = model.Tag(name=u'russian')
tag2 = model.Tag(name=u'tolstoy')
if auth_profile == "publisher":
pkg2.group = organization_group
# Flexible tag, allows spaces, upper-case,
# and all punctuation except commas
tag3 = model.Tag(name=u'Flexible \u30a1')
for obj in [pkg2, tag1, tag2, tag3]:
model.Session.add(obj)
pkg1.add_tags([tag1, tag2, tag3])
pkg2.add_tags([ tag1, tag3 ])
cls.tag_names = [ t.name for t in (tag1, tag2, tag3) ]
pkg1.license_id = u'other-open'
pkg2.license_id = u'cc-nc' # closed license
pkg2.title = u'A Wonderful Story'
pkg1.extras = {u'genre':'romantic novel',
u'original media':'book'}
# group
david = model.Group(name=u'david',
title=u'Dave\'s books',
description=u'These are books that David likes.',
type=auth_profile or 'group')
roger = model.Group(name=u'roger',
title=u'Roger\'s books',
description=u'Roger likes these books.',
type=auth_profile or 'group')
for obj in [david, roger]:
model.Session.add(obj)
cls.group_names.add(u'david')
cls.group_names.add(u'roger')
model.Session.flush()
model.Session.add(model.Member(table_id=pkg1.id, table_name='package', group=david))
model.Session.add(model.Member(table_id=pkg2.id, table_name='package', group=david))
model.Session.add(model.Member(table_id=pkg1.id, table_name='package', group=roger))
# authz
sysadmin = model.User(name=u'testsysadmin', password=u'testsysadmin')
sysadmin.sysadmin = True
model.Session.add_all([
model.User(name=u'tester', apikey=u'tester', password=u'tester'),
model.User(name=u'joeadmin', password=u'joeadmin'),
model.User(name=u'annafan', about=u'I love reading Annakarenina. My site: <a href="http://anna.com">anna.com</a>', password=u'annafan'),
model.User(name=u'russianfan', password=u'russianfan'),
sysadmin,
])
cls.user_refs.extend([u'tester', u'joeadmin', u'annafan', u'russianfan', u'testsysadmin'])
model.repo.commit_and_remove()
visitor = model.User.by_name(model.PSEUDO_USER__VISITOR)
anna = model.Package.by_name(u'annakarenina')
war = model.Package.by_name(u'warandpeace')
annafan = model.User.by_name(u'annafan')
russianfan = model.User.by_name(u'russianfan')
model.setup_default_user_roles(anna, [annafan])
model.setup_default_user_roles(war, [russianfan])
model.add_user_to_role(visitor, model.Role.ADMIN, war)
david = model.Group.by_name(u'david')
roger = model.Group.by_name(u'roger')
model.setup_default_user_roles(david, [russianfan])
model.setup_default_user_roles(roger, [russianfan])
model.add_user_to_role(visitor, model.Role.ADMIN, roger)
model.repo.commit_and_remove()
def do_publisher(cls, publisher_nid):
from ckan import model
from ckan.lib.munge import munge_title_to_name
log = global_log
pub = cls.get_cached_publisher_details(publisher_nid)
title = pub['title'].strip()
slug = munge_title_to_name(title)
g = model.Group.get(slug)
if g:
log.info('Found publisher in db: %s', g.name)
else:
cls.status.record('Not found in CKAN db', slug, do_print=False)
log.warn('Ignoring publisher that cannot be found in db: %s', slug)
return
if pub.get('parent_node'):
parent_pub_title = cls.get_cached_publisher_details(
pub['parent_node'])['title']
parent_name = munge_title_to_name(parent_pub_title)
parent = model.Group.get(parent_name)
if not parent:
cls.status.record('Cannot find parent in CKAN db',
g.name,
do_print=False)
log.warning('Cannot find parent %s of %s', parent_name,
pub.name)
return
existing_parents = [m.group for m in model.Session.query(model.Member).\
filter(model.Member.table_name=='group').\
filter(model.Member.table_id==g.id).\
filter(model.Member.state=='active')]
if existing_parents:
if len(existing_parents) > 1:
log.warn('Multiple parents for %s: %r', g.name,
[p.name for p in existing_parents])
if parent in existing_parents:
cls.status.record('Correct parent already',
g.name,
do_print=False)
log.info('Correct parent already: %s parent of %s',
parent.name, g.name)
return
else:
cls.status.record('Has another parent',
g.name,
do_print=False)
log.info(
'Has another parent: %r (instead of %s) parent of %s',
[p.name for p in existing_parents], parent.name,
g.name)
return
m = model.Member(group=parent, table_id=g.id, table_name='group')
model.Session.add(m)
model.Session.commit()
cls.status.record('Parent added', slug, do_print=False)
log.info('%s is made parent of %s', parent.name, g.name)
else:
log.info('%s has no parent in Drupal' % g.name)
cls.status.record('Has no parent in Drupal',
g.name,
do_print=False)
def create_groups(cls, group_dicts, admin_user_name=None, auth_profile=""):
'''A more featured interface for creating groups.
All group fields can be filled, packages added, can have
an admin user and be a member of other groups.'''
rev = model.repo.new_revision()
rev.author = cls.author
if admin_user_name:
admin_users = [model.User.by_name(admin_user_name)]
else:
admin_users = []
assert isinstance(group_dicts, (list, tuple))
group_attributes = set(('name', 'title', 'description', 'parent_id',
'type', 'is_organization'))
for group_dict in group_dicts:
if model.Group.by_name(unicode(group_dict['name'])):
log.warning('Cannot create group "%s" as it already exists.' %
group_dict['name'])
continue
pkg_names = group_dict.pop('packages', [])
group = model.Group(name=unicode(group_dict['name']))
group.type = auth_profile or 'group'
for key in group_dict:
if key in group_attributes:
setattr(group, key, group_dict[key])
elif key not in ('admins', 'editors', 'parent'):
group.extras[key] = group_dict[key]
assert isinstance(pkg_names, (list, tuple))
for pkg_name in pkg_names:
pkg = model.Package.by_name(unicode(pkg_name))
assert pkg, pkg_name
member = model.Member(group=group, table_id=pkg.id,
table_name='package')
model.Session.add(member)
model.Session.add(group)
admins = [model.User.by_name(user_name)
for user_name in group_dict.get('admins', [])] + \
admin_users
for admin in admins:
member = model.Member(group=group, table_id=admin.id,
table_name='user', capacity='admin')
model.Session.add(member)
editors = [model.User.by_name(user_name)
for user_name in group_dict.get('editors', [])]
for editor in editors:
member = model.Member(group=group, table_id=editor.id,
table_name='user', capacity='editor')
model.Session.add(member)
# Need to commit the current Group for two reasons:
# 1. It might have a parent, and the Member will need the Group.id
# value allocated on commit.
# 2. The next Group created may have this Group as a parent so
# creation of the Member needs to refer to this one.
model.Session.commit()
rev = model.repo.new_revision()
rev.author = cls.author
# add it to a parent's group
if 'parent' in group_dict:
parent = model.Group.by_name(unicode(group_dict['parent']))
assert parent, group_dict['parent']
member = model.Member(group=group, table_id=parent.id,
table_name='group', capacity='parent')
model.Session.add(member)
#model.setup_default_user_roles(group, admin_users)
cls.group_names.add(group_dict['name'])
model.repo.commit_and_remove()
def create_arbitrary(cls,
package_dicts: Union[dict[str, Any], list[dict[str,
Any]]],
relationships: list[dict[str, Any]] = [],
extra_user_names: list[str] = [],
extra_group_names: list[str] = []):
'''Creates packages and a few extra objects as well at the
same time if required.
@param package_dicts - a list of dictionaries with the package
properties.
Extra keys allowed:
@param extra_group_names - a list of group names to create. No
properties get set though.
'''
assert isinstance(relationships, (list, tuple))
assert isinstance(extra_user_names, (list, tuple))
assert isinstance(extra_group_names, (list, tuple))
model.Session.remove()
new_user_names = extra_user_names
new_group_names = set()
new_groups = {}
if package_dicts:
if isinstance(package_dicts, dict):
package_dicts = [package_dicts]
for item in package_dicts:
pkg_dict = {}
for field in cls.pkg_core_fields:
if field in item:
pkg_dict[field] = str(item[field])
if model.Package.by_name(pkg_dict['name']):
log.warning('Cannot create package "%s" as it already exists.' % \
(pkg_dict['name']))
continue
pkg = model.Package(**pkg_dict)
model.Session.add(pkg)
for attr, val in item.items():
if isinstance(val, str):
val = str(val)
if attr == 'name':
continue
if attr in cls.pkg_core_fields:
pass
elif attr == 'download_url':
pkg.add_resource(str(val))
elif attr == 'resources':
assert isinstance(val, (list, tuple))
for res_dict in val:
non_extras = {}
for k, v in res_dict.items():
if k != 'extras':
if not isinstance(v, datetime.datetime):
v = str(v)
non_extras[str(k)] = v
extras = {
str(k): str(v)
for k, v in res_dict.get('extras', {}).items()
}
pkg.add_resource(extras=extras, **non_extras)
elif attr == 'tags':
if isinstance(val, str):
tags = val.split()
elif isinstance(val, list):
tags = val
else:
raise NotImplementedError
for tag_name in tags:
tag_name = str(tag_name)
tag = model.Tag.by_name(tag_name)
if not tag:
tag = model.Tag(name=tag_name)
cls.tag_names.append(tag_name)
model.Session.add(tag)
pkg.add_tag(tag)
model.Session.flush()
elif attr == 'groups':
model.Session.flush()
if isinstance(val, str):
group_names = val.split()
elif isinstance(val, list):
group_names = val
else:
raise NotImplementedError
for group_name in group_names:
group = model.Group.by_name(str(group_name))
if not group:
if not group_name in new_groups:
group = model.Group(name=str(group_name))
model.Session.add(group)
new_group_names.add(group_name)
new_groups[group_name] = group
else:
# If adding multiple packages with the same
# group name, model.Group.by_name will not
# find the group as the session has not yet
# been committed at this point. Fetch from
# the new_groups dict instead.
group = new_groups[group_name]
capacity = 'organization' if group.is_organization\
else 'public'
member = model.Member(group=group,
table_id=pkg.id,
table_name='package',
capacity=capacity)
model.Session.add(member)
if group.is_organization:
pkg.owner_org = group.id
elif attr == 'license':
pkg.license_id = val
elif attr == 'license_id':
pkg.license_id = val
elif attr == 'extras':
pkg.extras = val
elif attr == 'admins':
assert 0, 'Deprecated param "admins"'
else:
raise NotImplementedError(attr)
cls.pkg_names.append(item['name'])
model.repo.commit_and_remove()
needs_commit = False
for group_name in extra_group_names:
group = model.Group(name=str(group_name))
model.Session.add(group)
new_group_names.add(group_name)
needs_commit = True
if needs_commit:
model.repo.commit_and_remove()
needs_commit = False
# create users that have been identified as being needed
for user_name in new_user_names:
if not model.User.by_name(str(user_name)):
user = model.User(name=str(user_name))
model.Session.add(user)
cls.user_refs.append(user_name)
needs_commit = True
if needs_commit:
model.repo.commit_and_remove()
needs_commit = False
# setup authz for groups just created
for group_name in new_group_names:
group = model.Group.by_name(str(group_name))
cls.group_names.add(group_name)
needs_commit = True
if needs_commit:
model.repo.commit_and_remove()
needs_commit = False
if relationships:
def get_pkg(pkg_name: str):
pkg = model.Package.by_name(str(pkg_name))
assert pkg
return pkg
for subject_name, relationship, object_name in relationships:
get_pkg(subject_name).add_relationship(str(relationship),
get_pkg(object_name))
needs_commit = True
model.repo.commit_and_remove()
def create(cls,
auth_profile: str = "",
package_type: Optional[str] = None):
model.Session.remove()
cls.pkg_names = [u'annakarenina', u'warandpeace']
pkg1 = model.Package(name=cls.pkg_names[0], type=package_type)
model.Session.add(pkg1)
pkg1.title = u'A Novel By Tolstoy'
pkg1.version = u'0.7a'
pkg1.url = u'http://datahub.io'
# put an & in the url string to test escaping
if 'alt_url' in model.Resource.get_extra_columns():
configured_extras = ({
'alt_url': u'alt123'
}, {
'alt_url': u'alt345'
})
else:
configured_extras = ({}, {})
pr1 = model.Resource(
url=u'http://datahub.io/download/x=1&y=2',
format=u'plain text',
description=u'Full text. Needs escaping: " Umlaut: \xfc',
hash=u'abc123',
extras={'size_extra': u'123'},
**configured_extras[0])
pr2 = model.Resource(url=u'http://datahub.io/index.json',
format=u'JSON',
description=u'Index of the novel',
hash=u'def456',
extras={'size_extra': u'345'},
**configured_extras[1])
model.Session.add(pr1)
model.Session.add(pr2)
pkg1.resources_all.append(pr1)
pkg1.resources_all.append(pr2)
pkg1.notes = u'''Some test notes
### A 3rd level heading
**Some bolded text.**
*Some italicized text.*
Foreign characters:
u with umlaut \xfc
66-style quote \u201c
foreign word: th\xfcmb
Needs escaping:
left arrow <
<http://ckan.net/>
'''
pkg2 = model.Package(name=cls.pkg_names[1], type=package_type)
tag1 = model.Tag(name=u'russian')
tag2 = model.Tag(name=u'tolstoy')
# Flexible tag, allows spaces, upper-case,
# and all punctuation except commas
tag3 = model.Tag(name=u'Flexible \u30a1')
for obj in [pkg2, tag1, tag2, tag3]:
model.Session.add(obj)
pkg1.add_tags([tag1, tag2, tag3])
pkg2.add_tags([tag1, tag3])
cls.tag_names = [t.name for t in (tag1, tag2, tag3)]
pkg1.license_id = u'other-open'
pkg2.license_id = u'cc-nc' # closed license
pkg2.title = u'A Wonderful Story'
pkg1.extras = {u'genre': 'romantic novel', u'original media': 'book'}
# group
david = model.Group(name=u'david',
title=u'Dave\'s books',
description=u'These are books that David likes.',
type=auth_profile or 'group')
roger = model.Group(name=u'roger',
title=u'Roger\'s books',
description=u'Roger likes these books.',
type=auth_profile or 'group')
for obj in [david, roger]:
model.Session.add(obj)
cls.group_names.add(u'david')
cls.group_names.add(u'roger')
model.Session.flush()
model.Session.add(
model.Member(table_id=pkg1.id, table_name='package', group=david))
model.Session.add(
model.Member(table_id=pkg2.id, table_name='package', group=david))
model.Session.add(
model.Member(table_id=pkg1.id, table_name='package', group=roger))
# authz
sysadmin = model.User(name=u'testsysadmin', password=u'testsysadmin')
sysadmin.sysadmin = True
model.Session.add_all([
model.User(name=u'tester', password=u'tester'),
model.User(name=u'joeadmin', password=u'joeadmin'),
model.User(
name=u'annafan',
about=
u'I love reading Annakarenina. My site: http://datahub.io',
password=u'annafan'),
model.User(name=u'russianfan', password=u'russianfan'),
sysadmin,
])
cls.user_refs.extend([
u'tester', u'joeadmin', u'annafan', u'russianfan', u'testsysadmin'
])
# Create activities for packages
for item in [pkg1, pkg2]:
activity = item.activity_stream_item('new', 'not logged in')
model.Session.add(activity)
model.repo.commit_and_remove()
def member_create(context, data_dict=None):
'''Make an object (e.g. a user, dataset or group) a member of a group.
If the object is already a member of the group then the capacity of the
membership will be updated.
You must be authorized to edit the group.
:param user: the user object
:type user: user object
:param id: the id or name of the group to add the object to
:type id: string
:param object: the id or name of the object to add
:type object: string
:param object_type: the type of the object being added, e.g. ``'package'``
or ``'user'``
:type object_type: string
:param capacity: the capacity of the membership
:type capacity: string
:returns: the newly created (or updated) membership
:rtype: dictionary
'''
from ckan import model
import ckan.plugins.toolkit as toolkit
import ckan.logic as logic
user, group_id, obj_id, obj_type, capacity = \
toolkit.get_or_bust(data_dict, ['user', 'id', 'object', 'object_type', 'capacity'])
rev = model.repo.new_revision()
#rev.author = user
#rev.message = 'Drupal DONL Catalogus'
group = model.Group.get(group_id)
if not group:
raise NotFound('Group was not found.')
obj_class = logic.model_name_to_class(model, obj_type)
obj = obj_class.get(obj_id)
if not obj:
raise NotFound('%s was not found.' % obj_type.title())
# Look up existing, in case it exists
member = model.Session.query(model.Member).\
filter(model.Member.table_name == obj_type).\
filter(model.Member.table_id == obj.id).\
filter(model.Member.group_id == group.id).\
filter(model.Member.state == 'active').first()
if not member:
member = model.Member(table_name=obj_type,
table_id=obj.id,
group_id=group.id,
state='active')
member.capacity = capacity
model.Session.add(member)
model.repo.commit()
log.debug('Username %s added as admin in organization %s', user.name,
group.name)
def _create_member_request(context, data_dict):
""" Helper to create member request """
role = data_dict.get('role', None)
if not role:
raise logic.NotFound
group = model.Group.get(data_dict.get('group', None))
if not group or group.type != 'organization':
raise logic.NotFound
user = context.get('user', None)
if authz.is_sysadmin(user):
raise logic.ValidationError({}, {_("Role"): _(
"As a sysadmin, you already have access to all organizations")})
userobj = model.User.get(user)
member = model.Session.query(model.Member).filter(model.Member.table_name == "user").filter(model.Member.table_id == userobj.id) \
.filter(model.Member.group_id == group.id).first()
# If there is a member for this organization and it is NOT deleted. Reuse
# existing if deleted
if member:
if member.state == 'pending':
message = _(
"You already have a pending request to the organization")
elif member.state == 'active':
message = _("You are already part of the organization")
# Unknown status. Should never happen..
elif member.state != 'deleted':
raise logic.ValidationError({"organization": _(
"Duplicate organization request")}, {_("Organization"): message})
else:
member = model.Member(table_name="user", table_id=userobj.id,
group_id=group.id, capacity=role, state='pending')
# TODO: Is there a way to get language associated to all admins. User table there is nothing as such stored
locale = get_safe_locale()
member.state = 'pending'
member.capacity = role
revision = model.repo.new_revision()
revision.author = user
revision.message = u'New member request'
model.Session.add(member)
# We need to flush since we need membership_id (member.id) already
model.Session.flush()
memberRequest = MemberRequest(
membership_id=member.id, role=role, status="pending", language=locale)
model.Session.add(memberRequest)
model.repo.commit()
url = config.get('ckan.site_url', "")
if url:
url = url + url_for('member_request_show', mrequest_id=member.id)
# Locale should be admin locale since mail is sent to admins
if role == 'admin':
for admin in _get_ckan_admins():
mail_new_membership_request(
locale, admin, group.display_name, url, userobj.display_name, userobj.email)
else:
for admin in _get_organization_admins(group.id):
mail_new_membership_request(
locale, admin, group.display_name, url, userobj.display_name, userobj.email)
return member
def add_publisher(cls, publisher_nid):
from ckan import model
from ckan.lib.munge import munge_title_to_name
if int(publisher_nid) in ignore_publishers:
global_log.info('Publisher ignored: %s (%s)', publisher_nid,
cls.get_cached_publisher_details(publisher_nid))
return
pub = cls.get_cached_publisher_details(publisher_nid)
title = pub['title'].strip()
slug = munge_title_to_name(title)
g = model.Group.get(slug)
if g:
global_log.info('Publisher already exists in db: %s', slug)
else:
g = model.Group(name=slug)
model.Session.add(g)
g.title = title
g.type = 'publisher'
g.description = pub['body']
field_pub_web_title = pub['field_pub_web'][0]['title'] if pub[
'field_pub_web'] else ''
g.extras[
'contact-name'] = '%s contact' % field_pub_web_title if field_pub_web_title else ''
g.extras['contact-email'] = pub['field_pub_email_display'][0][
'email'] if pub['field_pub_email_display'] else ''
g.extras['contact-phone'] = ''
g.extras['foi-name'] = ''
g.extras['foi-email'] = ''
g.extras['foi-phone'] = ''
acronym = pub['field_acronym'][0]['value'] if pub[
'field_acronym'] else ''
g.extras['abbreviation'] = acronym or ''
g.extras['website-url'] = (pub['field_pub_web'][0]['url']
or '') if pub['field_pub_web'] else ''
g.extras['website-name'] = (pub['field_pub_web'][0]['title']
or '') if pub['field_pub_web'] else ''
model.Session.commit()
title_and_abbreviation = '%s (%s)' % (title,
acronym) if acronym else title
global_log.info('Added/edited publisher: %s <%s>',
title_and_abbreviation, publisher_nid)
if pub.get('parent_node'):
parent_pub_title = cls.get_cached_publisher_details(
pub['parent_node'])['title']
parent = model.Group.get(munge_title_to_name(parent_pub_title))
if not parent:
parent = cls.add_publisher(pub['parent_node'])
if model.Session.query(model.Member).\
filter(model.Member.group==parent).\
filter(model.Member.table_id==g.id).count() == 0:
m = model.Member(group=parent,
table_id=g.id,
table_name='group')
model.Session.add(m)
global_log.info('%s is parent of %s', parent.name, g.name)
else:
global_log.info('%s is already a parent of %s', parent.name,
g.name)
model.Session.commit()
return g
def _create_member_request(context, data_dict):
""" Helper to create member request """
changed = False
role = data_dict['role']
group = model.Group.get(data_dict['group'])
if not group or group.type != 'organization':
raise NotFound
user = context['user']
if new_authz.is_sysadmin(user):
raise ValidationError({}, {_("Role"): _("As a sysadmin, you already have access to all organizations")})
userobj = model.User.get(user)
member = model.Session.query(model.Member).filter(model.Member.table_name == "user").filter(
model.Member.table_id == userobj.id) \
.filter(model.Member.group_id == group.id).filter(
or_(model.Member.state == 'active', model.Member.state == 'pending')).first()
if member:
if member.state == 'pending':
message = _("You already have a pending request to the organization")
else:
message = _("You are already part of the organization")
raise ValidationError({"organization": _("Duplicate organization request")}, {_("Organization"): message})
member = model.Session.query(model.Member).filter(model.Member.table_name == "user").filter(
model.Member.table_id == userobj.id) \
.filter(model.Member.group_id == group.id).first()
if not member:
member = model.Member(table_name="user", table_id=userobj.id, group_id=group.id, capacity=role, state='pending')
changed = True
locale = _get_safe_locale()
if member.state != 'pending' or changed:
member.state = 'pending'
member.capacity = role
revision = model.repo.new_revision()
revision.author = user
if 'message' in context:
revision.message = context['message']
elif changed:
revision.message = u'New member request'
else:
revision.message = u'Changed member request'
if changed:
model.Session.add(member)
else:
member.save()
extra = MemberExtra(member_id=member.id, key="locale", value=locale)
extra.save()
model.repo.commit()
changed = True
url = config.get('ckan.site_url', "")
if url:
url = url + url_for('member_request_show', member_id=member.id)
admin_list = []
if role == 'admin':
for admin in get_ckan_admins():
admin_list.append({'display_name': admin.display_name, 'email': admin.email})
# _mail_new_membership_request(locale, admin, group, url, userobj, data_dict)
else:
for admin in get_organization_admins(group.id):
admin_list.append({'display_name': admin.display_name, 'email': admin.email})
# _mail_new_membership_request(locale, admin, group, url, userobj, data_dict)
_mail_new_membership_request(locale, None, group, url, userobj, data_dict, admin_list)
return member, changed
def package_membership_list_save(group_dicts: Optional[list[dict[str, Any]]],
package: 'model.Package',
context: Context) -> None:
allow_partial_update = context.get("allow_partial_update", False)
if group_dicts is None and allow_partial_update:
return
capacity = 'public'
model = context["model"]
session = context["session"]
user = context.get('user', '')
members = session.query(model.Member) \
.filter(model.Member.table_id == package.id) \
.filter(model.Member.capacity != 'organization')
group_member = dict((member.group, member) for member in members)
groups: set[model.Group] = set()
for group_dict in group_dicts or []:
id = group_dict.get("id")
name = group_dict.get("name")
capacity = group_dict.get("capacity", "public")
if capacity == 'organization':
continue
if id:
group = session.query(model.Group).get(id)
else:
group = session.query(model.Group).filter_by(name=name).first()
if group:
groups.add(group)
## need to flush so we can get out the package id
model.Session.flush()
# Remove any groups we are no longer in
for group in set(group_member.keys()) - groups:
member_obj = group_member[group]
if member_obj and member_obj.state == 'deleted':
continue
if authz.has_user_permission_for_group_or_org(member_obj.group_id,
user, 'read'):
member_obj.capacity = capacity
member_obj.state = 'deleted'
session.add(member_obj)
# Add any new groups
for group in groups:
member_obj = group_member.get(group)
if member_obj and member_obj.state == 'active':
continue
if authz.has_user_permission_for_group_or_org(group.id, user, 'read'):
member_obj = group_member.get(group)
if member_obj:
member_obj.capacity = capacity
member_obj.state = 'active'
else:
member_obj = model.Member(table_id=package.id,
table_name='package',
group=group,
capacity=capacity,
group_id=group.id,
state='active')
session.add(member_obj)
def nsw_package_membership_list_save(group_dicts, package, context):
allow_partial_update = context.get("allow_partial_update", False)
if group_dicts is None and allow_partial_update:
return
capacity = 'public'
model = context["model"]
session = context["session"]
user = context.get('user')
members = session.query(model.Member) \
.filter(model.Member.table_id == package.id) \
.filter(model.Member.capacity != 'organization')
group_member = dict((member.group, member) for member in members)
groups = set()
for group_dict in group_dicts or []:
id = group_dict.get("id")
name = group_dict.get("name")
capacity = group_dict.get("capacity", "public")
if capacity == 'organization':
continue
if id:
group = session.query(model.Group).get(id)
else:
group = session.query(model.Group).filter_by(name=name).first()
if group:
groups.add(group)
## need to flush so we can get out the package id
model.Session.flush()
# Remove any groups we are no longer in
for group in set(group_member.keys()) - groups:
member_obj = group_member[group]
if member_obj and member_obj.state == 'deleted':
continue
# Bypass authorization to enable datasets to be removed from AGIFT classification
member_obj.capacity = capacity
member_obj.state = 'deleted'
session.add(member_obj)
# Add any new groups
for group in groups:
member_obj = group_member.get(group)
if member_obj and member_obj.state == 'active':
continue
# Bypass authorization to enable datasets to be added to AGIFT classification
member_obj = group_member.get(group)
if member_obj:
member_obj.capacity = capacity
member_obj.state = 'active'
else:
member_obj = model.Member(table_id=package.id,
table_name='package',
group=group,
capacity=capacity,
group_id=group.id,
state='active')
session.add(member_obj)
def command(config_ini, nodepublisher_csv, users_csv):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
from ckan import model
from ckan.lib.munge import munge_title_to_name
import ckanext.dgu.plugin
logging.config.fileConfig(config_ini_filepath)
global log
log = logging.getLogger(os.path.basename(__file__))
model.init_model(engine)
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
with open(nodepublisher_csv, 'rU') as f:
reader = csv.reader(f)
for row in reader:
publishers[int(row[0])] = munge_title_to_name(row[1])
log.info('Opened list of %i publishers', reader.line_num)
# get rid of flash message warnings
warnings_.filterwarnings('ignore', '.*flash message.*')
ckanext.dgu.plugin.log.disabled = True
with open(users_csv, 'rU') as f:
reader = csv.reader(f)
for row in reader:
model.repo.new_revision()
node_id, name, email, publisher_id = row
# create a new user
uname, user_id = _add_new_user(int(node_id), name, email)
if not user_id:
# validation error. warning already printed
continue
# Find the publisher and add them as editor
if node_id:
publisher_name = publishers[int(publisher_id)]
publisher = model.Group.by_name(publisher_name)
if not publisher:
warn(
'Could not find publisher %r so skipping making %r editor for it.',
publisher_name, name)
continue
capacity = 'editor'
# Check for Member where table_name is u['id']
res = model.Session.query(model.Member).\
from_statement(MEMBER_LOOKUP).\
params(userid=user_id, groupid=publisher.id).all()
if len(res) == 0:
m = model.Member(group_id=publisher.id,
table_id=user_id,
table_name='user',
capacity=capacity)
model.Session.add(m)
log.info('Made %r editor for %r', name, publisher_name)
else:
log.info('%r already editor for %r', name, publisher_name)
# Update harvest_source user_id field to new user id.
model.Session.execute(HARVEST_QUERY,
params={
'uid': user_id,
'node_id': str(node_id)
})
model.Session.commit()
log.info('Processed list of %i users', reader.line_num)
log.info('Warnings (%i): %r', len(warnings), warnings)
