def create_arbitrary(cls, package_dicts, relationships=[], extra_user_names=[], extra_group_names=[], admins=[]): '''Creates packages and a few extra objects as well at the same time if required. @param package_dicts - a list of dictionaries with the package properties. Extra keys allowed: "admins" - list of user names to make admin for this package. @param extra_group_names - a list of group names to create. No properties get set though. @param admins - a list of user names to make admins of all the packages created. ''' assert isinstance(relationships, (list, tuple)) assert isinstance(extra_user_names, (list, tuple)) assert isinstance(extra_group_names, (list, tuple)) model.Session.remove() new_user_names = extra_user_names new_group_names = set() new_groups = {} rev = model.repo.new_revision() rev.author = cls.author rev.message = u'Creating test packages.' admins_list = defaultdict(list) # package_name: admin_names if package_dicts: if isinstance(package_dicts, dict): package_dicts = [package_dicts] for item in package_dicts: pkg_dict = {} for field in cls.pkg_core_fields: if item.has_key(field): pkg_dict[field] = unicode(item[field]) if model.Package.by_name(pkg_dict['name']): log.warning('Cannot create package "%s" as it already exists.' % \ (pkg_dict['name'])) continue pkg = model.Package(**pkg_dict) model.Session.add(pkg) for attr, val in item.items(): if isinstance(val, str): val = unicode(val) if attr=='name': continue if attr in cls.pkg_core_fields: pass elif attr == 'download_url': pkg.add_resource(unicode(val)) elif attr == 'resources': assert isinstance(val, (list, tuple)) for res_dict in val: non_extras = {} for k, v in res_dict.items(): if k != 'extras': if not isinstance(v, datetime.datetime): v = unicode(v) non_extras[str(k)] = v extras = dict([(str(k), unicode(v)) for k, v in res_dict.get('extras', {}).items()]) pkg.add_resource(extras=extras, **non_extras) elif attr == 'tags': if isinstance(val, (str, unicode)): tags = val.split() elif isinstance(val, list): tags = val else: raise NotImplementedError for tag_name in tags: tag_name = unicode(tag_name) tag = model.Tag.by_name(tag_name) if not tag: tag = model.Tag(name=tag_name) cls.tag_names.append(tag_name) model.Session.add(tag) pkg.add_tag(tag) model.Session.flush() elif attr == 'groups': model.Session.flush() if isinstance(val, (str, unicode)): group_names = val.split() elif isinstance(val, list): group_names = val else: raise NotImplementedError for group_name in group_names: group = model.Group.by_name(unicode(group_name)) if not group: if not group_name in new_groups: group = model.Group(name=unicode(group_name)) model.Session.add(group) new_group_names.add(group_name) new_groups[group_name] = group else: # If adding multiple packages with the same group name, # model.Group.by_name will not find the group as the # session has not yet been committed at this point. # Fetch from the new_groups dict instead. group = new_groups[group_name] member = model.Member(group=group, table_id=pkg.id, table_name='package') model.Session.add(member) elif attr == 'license': pkg.license_id = val elif attr == 'license_id': pkg.license_id = val elif attr == 'extras': pkg.extras = val elif attr == 'admins': assert isinstance(val, list) admins_list[item['name']].extend(val) for user_name in val: if user_name not in new_user_names: new_user_names.append(user_name) else: raise NotImplementedError(attr) cls.pkg_names.append(item['name']) model.setup_default_user_roles(pkg, admins=[]) for admin in admins: admins_list[item['name']].append(admin) model.repo.commit_and_remove() needs_commit = False rev = model.repo.new_revision() for group_name in extra_group_names: group = model.Group(name=unicode(group_name)) model.Session.add(group) new_group_names.add(group_name) needs_commit = True if needs_commit: model.repo.commit_and_remove() needs_commit = False # create users that have been identified as being needed for user_name in new_user_names: if not model.User.by_name(unicode(user_name)): user = model.User(name=unicode(user_name)) model.Session.add(user) cls.user_refs.append(user_name) needs_commit = True if needs_commit: model.repo.commit_and_remove() needs_commit = False # setup authz for admins for pkg_name, admins in admins_list.items(): pkg = model.Package.by_name(unicode(pkg_name)) admins_obj_list = [] for admin in admins: if isinstance(admin, model.User): admin_obj = admin else: admin_obj = model.User.by_name(unicode(admin)) assert admin_obj, admin admins_obj_list.append(admin_obj) model.setup_default_user_roles(pkg, admins_obj_list) needs_commit = True # setup authz for groups just created for group_name in new_group_names: group = model.Group.by_name(unicode(group_name)) model.setup_default_user_roles(group) cls.group_names.add(group_name) needs_commit = True if needs_commit: model.repo.commit_and_remove() needs_commit = False if relationships: rev = model.repo.new_revision() rev.author = cls.author rev.message = u'Creating package relationships.' def pkg(pkg_name): return model.Package.by_name(unicode(pkg_name)) for subject_name, relationship, object_name in relationships: pkg(subject_name).add_relationship( unicode(relationship), pkg(object_name)) needs_commit = True model.repo.commit_and_remove()
def create(cls, auth_profile="", package_type=None): model.Session.remove() rev = model.repo.new_revision() # same name as user we create below rev.author = cls.author rev.message = u'''Creating test data. * Package: annakarenina * Package: warandpeace * Associated tags, etc etc ''' if auth_profile == "publisher": organization_group = model.Group(name=u"organization_group", type="organization") cls.pkg_names = [u'annakarenina', u'warandpeace'] pkg1 = model.Package(name=cls.pkg_names[0], type=package_type) if auth_profile == "publisher": pkg1.group = organization_group model.Session.add(pkg1) pkg1.title = u'A Novel By Tolstoy' pkg1.version = u'0.7a' pkg1.url = u'http://www.annakarenina.com' # put an & in the url string to test escaping if 'alt_url' in model.Resource.get_extra_columns(): configured_extras = ({'alt_url': u'alt123'}, {'alt_url': u'alt345'}) else: configured_extras = ({}, {}) pr1 = model.Resource( url=u'http://www.annakarenina.com/download/x=1&y=2', format=u'plain text', description=u'Full text. Needs escaping: " Umlaut: \xfc', hash=u'abc123', extras={'size_extra': u'123'}, **configured_extras[0] ) pr2 = model.Resource( url=u'http://www.annakarenina.com/index.json', format=u'json', description=u'Index of the novel', hash=u'def456', extras={'size_extra': u'345'}, **configured_extras[1] ) model.Session.add(pr1) model.Session.add(pr2) pkg1.resource_groups_all[0].resources_all.append(pr1) pkg1.resource_groups_all[0].resources_all.append(pr2) pkg1.notes = u'''Some test notes ### A 3rd level heading **Some bolded text.** *Some italicized text.* Foreign characters: u with umlaut \xfc 66-style quote \u201c foreign word: th\xfcmb Needs escaping: left arrow < <http://ckan.net/> ''' pkg2 = model.Package(name=cls.pkg_names[1], type=package_type) tag1 = model.Tag(name=u'russian') tag2 = model.Tag(name=u'tolstoy') if auth_profile == "publisher": pkg2.group = organization_group # Flexible tag, allows spaces, upper-case, # and all punctuation except commas tag3 = model.Tag(name=u'Flexible \u30a1') for obj in [pkg2, tag1, tag2, tag3]: model.Session.add(obj) pkg1.add_tags([tag1, tag2, tag3]) pkg2.add_tags([ tag1, tag3 ]) cls.tag_names = [ t.name for t in (tag1, tag2, tag3) ] pkg1.license_id = u'other-open' pkg2.license_id = u'cc-nc' # closed license pkg2.title = u'A Wonderful Story' pkg1.extras = {u'genre':'romantic novel', u'original media':'book'} # group david = model.Group(name=u'david', title=u'Dave\'s books', description=u'These are books that David likes.', type=auth_profile or 'group') roger = model.Group(name=u'roger', title=u'Roger\'s books', description=u'Roger likes these books.', type=auth_profile or 'group') for obj in [david, roger]: model.Session.add(obj) cls.group_names.add(u'david') cls.group_names.add(u'roger') model.Session.flush() model.Session.add(model.Member(table_id=pkg1.id, table_name='package', group=david)) model.Session.add(model.Member(table_id=pkg2.id, table_name='package', group=david)) model.Session.add(model.Member(table_id=pkg1.id, table_name='package', group=roger)) # authz sysadmin = model.User(name=u'testsysadmin', password=u'testsysadmin') sysadmin.sysadmin = True model.Session.add_all([ model.User(name=u'tester', apikey=u'tester', password=u'tester'), model.User(name=u'joeadmin', password=u'joeadmin'), model.User(name=u'annafan', about=u'I love reading Annakarenina. My site: <a href="http://anna.com">anna.com</a>', password=u'annafan'), model.User(name=u'russianfan', password=u'russianfan'), sysadmin, ]) cls.user_refs.extend([u'tester', u'joeadmin', u'annafan', u'russianfan', u'testsysadmin']) model.repo.commit_and_remove() visitor = model.User.by_name(model.PSEUDO_USER__VISITOR) anna = model.Package.by_name(u'annakarenina') war = model.Package.by_name(u'warandpeace') annafan = model.User.by_name(u'annafan') russianfan = model.User.by_name(u'russianfan') model.setup_default_user_roles(anna, [annafan]) model.setup_default_user_roles(war, [russianfan]) model.add_user_to_role(visitor, model.Role.ADMIN, war) david = model.Group.by_name(u'david') roger = model.Group.by_name(u'roger') model.setup_default_user_roles(david, [russianfan]) model.setup_default_user_roles(roger, [russianfan]) model.add_user_to_role(visitor, model.Role.ADMIN, roger) model.repo.commit_and_remove()
def do_publisher(cls, publisher_nid): from ckan import model from ckan.lib.munge import munge_title_to_name log = global_log pub = cls.get_cached_publisher_details(publisher_nid) title = pub['title'].strip() slug = munge_title_to_name(title) g = model.Group.get(slug) if g: log.info('Found publisher in db: %s', g.name) else: cls.status.record('Not found in CKAN db', slug, do_print=False) log.warn('Ignoring publisher that cannot be found in db: %s', slug) return if pub.get('parent_node'): parent_pub_title = cls.get_cached_publisher_details( pub['parent_node'])['title'] parent_name = munge_title_to_name(parent_pub_title) parent = model.Group.get(parent_name) if not parent: cls.status.record('Cannot find parent in CKAN db', g.name, do_print=False) log.warning('Cannot find parent %s of %s', parent_name, pub.name) return existing_parents = [m.group for m in model.Session.query(model.Member).\ filter(model.Member.table_name=='group').\ filter(model.Member.table_id==g.id).\ filter(model.Member.state=='active')] if existing_parents: if len(existing_parents) > 1: log.warn('Multiple parents for %s: %r', g.name, [p.name for p in existing_parents]) if parent in existing_parents: cls.status.record('Correct parent already', g.name, do_print=False) log.info('Correct parent already: %s parent of %s', parent.name, g.name) return else: cls.status.record('Has another parent', g.name, do_print=False) log.info( 'Has another parent: %r (instead of %s) parent of %s', [p.name for p in existing_parents], parent.name, g.name) return m = model.Member(group=parent, table_id=g.id, table_name='group') model.Session.add(m) model.Session.commit() cls.status.record('Parent added', slug, do_print=False) log.info('%s is made parent of %s', parent.name, g.name) else: log.info('%s has no parent in Drupal' % g.name) cls.status.record('Has no parent in Drupal', g.name, do_print=False)
def create_groups(cls, group_dicts, admin_user_name=None, auth_profile=""): '''A more featured interface for creating groups. All group fields can be filled, packages added, can have an admin user and be a member of other groups.''' rev = model.repo.new_revision() rev.author = cls.author if admin_user_name: admin_users = [model.User.by_name(admin_user_name)] else: admin_users = [] assert isinstance(group_dicts, (list, tuple)) group_attributes = set(('name', 'title', 'description', 'parent_id', 'type', 'is_organization')) for group_dict in group_dicts: if model.Group.by_name(unicode(group_dict['name'])): log.warning('Cannot create group "%s" as it already exists.' % group_dict['name']) continue pkg_names = group_dict.pop('packages', []) group = model.Group(name=unicode(group_dict['name'])) group.type = auth_profile or 'group' for key in group_dict: if key in group_attributes: setattr(group, key, group_dict[key]) elif key not in ('admins', 'editors', 'parent'): group.extras[key] = group_dict[key] assert isinstance(pkg_names, (list, tuple)) for pkg_name in pkg_names: pkg = model.Package.by_name(unicode(pkg_name)) assert pkg, pkg_name member = model.Member(group=group, table_id=pkg.id, table_name='package') model.Session.add(member) model.Session.add(group) admins = [model.User.by_name(user_name) for user_name in group_dict.get('admins', [])] + \ admin_users for admin in admins: member = model.Member(group=group, table_id=admin.id, table_name='user', capacity='admin') model.Session.add(member) editors = [model.User.by_name(user_name) for user_name in group_dict.get('editors', [])] for editor in editors: member = model.Member(group=group, table_id=editor.id, table_name='user', capacity='editor') model.Session.add(member) # Need to commit the current Group for two reasons: # 1. It might have a parent, and the Member will need the Group.id # value allocated on commit. # 2. The next Group created may have this Group as a parent so # creation of the Member needs to refer to this one. model.Session.commit() rev = model.repo.new_revision() rev.author = cls.author # add it to a parent's group if 'parent' in group_dict: parent = model.Group.by_name(unicode(group_dict['parent'])) assert parent, group_dict['parent'] member = model.Member(group=group, table_id=parent.id, table_name='group', capacity='parent') model.Session.add(member) #model.setup_default_user_roles(group, admin_users) cls.group_names.add(group_dict['name']) model.repo.commit_and_remove()
def create_arbitrary(cls, package_dicts: Union[dict[str, Any], list[dict[str, Any]]], relationships: list[dict[str, Any]] = [], extra_user_names: list[str] = [], extra_group_names: list[str] = []): '''Creates packages and a few extra objects as well at the same time if required. @param package_dicts - a list of dictionaries with the package properties. Extra keys allowed: @param extra_group_names - a list of group names to create. No properties get set though. ''' assert isinstance(relationships, (list, tuple)) assert isinstance(extra_user_names, (list, tuple)) assert isinstance(extra_group_names, (list, tuple)) model.Session.remove() new_user_names = extra_user_names new_group_names = set() new_groups = {} if package_dicts: if isinstance(package_dicts, dict): package_dicts = [package_dicts] for item in package_dicts: pkg_dict = {} for field in cls.pkg_core_fields: if field in item: pkg_dict[field] = str(item[field]) if model.Package.by_name(pkg_dict['name']): log.warning('Cannot create package "%s" as it already exists.' % \ (pkg_dict['name'])) continue pkg = model.Package(**pkg_dict) model.Session.add(pkg) for attr, val in item.items(): if isinstance(val, str): val = str(val) if attr == 'name': continue if attr in cls.pkg_core_fields: pass elif attr == 'download_url': pkg.add_resource(str(val)) elif attr == 'resources': assert isinstance(val, (list, tuple)) for res_dict in val: non_extras = {} for k, v in res_dict.items(): if k != 'extras': if not isinstance(v, datetime.datetime): v = str(v) non_extras[str(k)] = v extras = { str(k): str(v) for k, v in res_dict.get('extras', {}).items() } pkg.add_resource(extras=extras, **non_extras) elif attr == 'tags': if isinstance(val, str): tags = val.split() elif isinstance(val, list): tags = val else: raise NotImplementedError for tag_name in tags: tag_name = str(tag_name) tag = model.Tag.by_name(tag_name) if not tag: tag = model.Tag(name=tag_name) cls.tag_names.append(tag_name) model.Session.add(tag) pkg.add_tag(tag) model.Session.flush() elif attr == 'groups': model.Session.flush() if isinstance(val, str): group_names = val.split() elif isinstance(val, list): group_names = val else: raise NotImplementedError for group_name in group_names: group = model.Group.by_name(str(group_name)) if not group: if not group_name in new_groups: group = model.Group(name=str(group_name)) model.Session.add(group) new_group_names.add(group_name) new_groups[group_name] = group else: # If adding multiple packages with the same # group name, model.Group.by_name will not # find the group as the session has not yet # been committed at this point. Fetch from # the new_groups dict instead. group = new_groups[group_name] capacity = 'organization' if group.is_organization\ else 'public' member = model.Member(group=group, table_id=pkg.id, table_name='package', capacity=capacity) model.Session.add(member) if group.is_organization: pkg.owner_org = group.id elif attr == 'license': pkg.license_id = val elif attr == 'license_id': pkg.license_id = val elif attr == 'extras': pkg.extras = val elif attr == 'admins': assert 0, 'Deprecated param "admins"' else: raise NotImplementedError(attr) cls.pkg_names.append(item['name']) model.repo.commit_and_remove() needs_commit = False for group_name in extra_group_names: group = model.Group(name=str(group_name)) model.Session.add(group) new_group_names.add(group_name) needs_commit = True if needs_commit: model.repo.commit_and_remove() needs_commit = False # create users that have been identified as being needed for user_name in new_user_names: if not model.User.by_name(str(user_name)): user = model.User(name=str(user_name)) model.Session.add(user) cls.user_refs.append(user_name) needs_commit = True if needs_commit: model.repo.commit_and_remove() needs_commit = False # setup authz for groups just created for group_name in new_group_names: group = model.Group.by_name(str(group_name)) cls.group_names.add(group_name) needs_commit = True if needs_commit: model.repo.commit_and_remove() needs_commit = False if relationships: def get_pkg(pkg_name: str): pkg = model.Package.by_name(str(pkg_name)) assert pkg return pkg for subject_name, relationship, object_name in relationships: get_pkg(subject_name).add_relationship(str(relationship), get_pkg(object_name)) needs_commit = True model.repo.commit_and_remove()
def create(cls, auth_profile: str = "", package_type: Optional[str] = None): model.Session.remove() cls.pkg_names = [u'annakarenina', u'warandpeace'] pkg1 = model.Package(name=cls.pkg_names[0], type=package_type) model.Session.add(pkg1) pkg1.title = u'A Novel By Tolstoy' pkg1.version = u'0.7a' pkg1.url = u'http://datahub.io' # put an & in the url string to test escaping if 'alt_url' in model.Resource.get_extra_columns(): configured_extras = ({ 'alt_url': u'alt123' }, { 'alt_url': u'alt345' }) else: configured_extras = ({}, {}) pr1 = model.Resource( url=u'http://datahub.io/download/x=1&y=2', format=u'plain text', description=u'Full text. Needs escaping: " Umlaut: \xfc', hash=u'abc123', extras={'size_extra': u'123'}, **configured_extras[0]) pr2 = model.Resource(url=u'http://datahub.io/index.json', format=u'JSON', description=u'Index of the novel', hash=u'def456', extras={'size_extra': u'345'}, **configured_extras[1]) model.Session.add(pr1) model.Session.add(pr2) pkg1.resources_all.append(pr1) pkg1.resources_all.append(pr2) pkg1.notes = u'''Some test notes ### A 3rd level heading **Some bolded text.** *Some italicized text.* Foreign characters: u with umlaut \xfc 66-style quote \u201c foreign word: th\xfcmb Needs escaping: left arrow < <http://ckan.net/> ''' pkg2 = model.Package(name=cls.pkg_names[1], type=package_type) tag1 = model.Tag(name=u'russian') tag2 = model.Tag(name=u'tolstoy') # Flexible tag, allows spaces, upper-case, # and all punctuation except commas tag3 = model.Tag(name=u'Flexible \u30a1') for obj in [pkg2, tag1, tag2, tag3]: model.Session.add(obj) pkg1.add_tags([tag1, tag2, tag3]) pkg2.add_tags([tag1, tag3]) cls.tag_names = [t.name for t in (tag1, tag2, tag3)] pkg1.license_id = u'other-open' pkg2.license_id = u'cc-nc' # closed license pkg2.title = u'A Wonderful Story' pkg1.extras = {u'genre': 'romantic novel', u'original media': 'book'} # group david = model.Group(name=u'david', title=u'Dave\'s books', description=u'These are books that David likes.', type=auth_profile or 'group') roger = model.Group(name=u'roger', title=u'Roger\'s books', description=u'Roger likes these books.', type=auth_profile or 'group') for obj in [david, roger]: model.Session.add(obj) cls.group_names.add(u'david') cls.group_names.add(u'roger') model.Session.flush() model.Session.add( model.Member(table_id=pkg1.id, table_name='package', group=david)) model.Session.add( model.Member(table_id=pkg2.id, table_name='package', group=david)) model.Session.add( model.Member(table_id=pkg1.id, table_name='package', group=roger)) # authz sysadmin = model.User(name=u'testsysadmin', password=u'testsysadmin') sysadmin.sysadmin = True model.Session.add_all([ model.User(name=u'tester', password=u'tester'), model.User(name=u'joeadmin', password=u'joeadmin'), model.User( name=u'annafan', about= u'I love reading Annakarenina. My site: http://datahub.io', password=u'annafan'), model.User(name=u'russianfan', password=u'russianfan'), sysadmin, ]) cls.user_refs.extend([ u'tester', u'joeadmin', u'annafan', u'russianfan', u'testsysadmin' ]) # Create activities for packages for item in [pkg1, pkg2]: activity = item.activity_stream_item('new', 'not logged in') model.Session.add(activity) model.repo.commit_and_remove()
def member_create(context, data_dict=None): '''Make an object (e.g. a user, dataset or group) a member of a group. If the object is already a member of the group then the capacity of the membership will be updated. You must be authorized to edit the group. :param user: the user object :type user: user object :param id: the id or name of the group to add the object to :type id: string :param object: the id or name of the object to add :type object: string :param object_type: the type of the object being added, e.g. ``'package'`` or ``'user'`` :type object_type: string :param capacity: the capacity of the membership :type capacity: string :returns: the newly created (or updated) membership :rtype: dictionary ''' from ckan import model import ckan.plugins.toolkit as toolkit import ckan.logic as logic user, group_id, obj_id, obj_type, capacity = \ toolkit.get_or_bust(data_dict, ['user', 'id', 'object', 'object_type', 'capacity']) rev = model.repo.new_revision() #rev.author = user #rev.message = 'Drupal DONL Catalogus' group = model.Group.get(group_id) if not group: raise NotFound('Group was not found.') obj_class = logic.model_name_to_class(model, obj_type) obj = obj_class.get(obj_id) if not obj: raise NotFound('%s was not found.' % obj_type.title()) # Look up existing, in case it exists member = model.Session.query(model.Member).\ filter(model.Member.table_name == obj_type).\ filter(model.Member.table_id == obj.id).\ filter(model.Member.group_id == group.id).\ filter(model.Member.state == 'active').first() if not member: member = model.Member(table_name=obj_type, table_id=obj.id, group_id=group.id, state='active') member.capacity = capacity model.Session.add(member) model.repo.commit() log.debug('Username %s added as admin in organization %s', user.name, group.name)
def _create_member_request(context, data_dict): """ Helper to create member request """ role = data_dict.get('role', None) if not role: raise logic.NotFound group = model.Group.get(data_dict.get('group', None)) if not group or group.type != 'organization': raise logic.NotFound user = context.get('user', None) if authz.is_sysadmin(user): raise logic.ValidationError({}, {_("Role"): _( "As a sysadmin, you already have access to all organizations")}) userobj = model.User.get(user) member = model.Session.query(model.Member).filter(model.Member.table_name == "user").filter(model.Member.table_id == userobj.id) \ .filter(model.Member.group_id == group.id).first() # If there is a member for this organization and it is NOT deleted. Reuse # existing if deleted if member: if member.state == 'pending': message = _( "You already have a pending request to the organization") elif member.state == 'active': message = _("You are already part of the organization") # Unknown status. Should never happen.. elif member.state != 'deleted': raise logic.ValidationError({"organization": _( "Duplicate organization request")}, {_("Organization"): message}) else: member = model.Member(table_name="user", table_id=userobj.id, group_id=group.id, capacity=role, state='pending') # TODO: Is there a way to get language associated to all admins. User table there is nothing as such stored locale = get_safe_locale() member.state = 'pending' member.capacity = role revision = model.repo.new_revision() revision.author = user revision.message = u'New member request' model.Session.add(member) # We need to flush since we need membership_id (member.id) already model.Session.flush() memberRequest = MemberRequest( membership_id=member.id, role=role, status="pending", language=locale) model.Session.add(memberRequest) model.repo.commit() url = config.get('ckan.site_url', "") if url: url = url + url_for('member_request_show', mrequest_id=member.id) # Locale should be admin locale since mail is sent to admins if role == 'admin': for admin in _get_ckan_admins(): mail_new_membership_request( locale, admin, group.display_name, url, userobj.display_name, userobj.email) else: for admin in _get_organization_admins(group.id): mail_new_membership_request( locale, admin, group.display_name, url, userobj.display_name, userobj.email) return member
def add_publisher(cls, publisher_nid): from ckan import model from ckan.lib.munge import munge_title_to_name if int(publisher_nid) in ignore_publishers: global_log.info('Publisher ignored: %s (%s)', publisher_nid, cls.get_cached_publisher_details(publisher_nid)) return pub = cls.get_cached_publisher_details(publisher_nid) title = pub['title'].strip() slug = munge_title_to_name(title) g = model.Group.get(slug) if g: global_log.info('Publisher already exists in db: %s', slug) else: g = model.Group(name=slug) model.Session.add(g) g.title = title g.type = 'publisher' g.description = pub['body'] field_pub_web_title = pub['field_pub_web'][0]['title'] if pub[ 'field_pub_web'] else '' g.extras[ 'contact-name'] = '%s contact' % field_pub_web_title if field_pub_web_title else '' g.extras['contact-email'] = pub['field_pub_email_display'][0][ 'email'] if pub['field_pub_email_display'] else '' g.extras['contact-phone'] = '' g.extras['foi-name'] = '' g.extras['foi-email'] = '' g.extras['foi-phone'] = '' acronym = pub['field_acronym'][0]['value'] if pub[ 'field_acronym'] else '' g.extras['abbreviation'] = acronym or '' g.extras['website-url'] = (pub['field_pub_web'][0]['url'] or '') if pub['field_pub_web'] else '' g.extras['website-name'] = (pub['field_pub_web'][0]['title'] or '') if pub['field_pub_web'] else '' model.Session.commit() title_and_abbreviation = '%s (%s)' % (title, acronym) if acronym else title global_log.info('Added/edited publisher: %s <%s>', title_and_abbreviation, publisher_nid) if pub.get('parent_node'): parent_pub_title = cls.get_cached_publisher_details( pub['parent_node'])['title'] parent = model.Group.get(munge_title_to_name(parent_pub_title)) if not parent: parent = cls.add_publisher(pub['parent_node']) if model.Session.query(model.Member).\ filter(model.Member.group==parent).\ filter(model.Member.table_id==g.id).count() == 0: m = model.Member(group=parent, table_id=g.id, table_name='group') model.Session.add(m) global_log.info('%s is parent of %s', parent.name, g.name) else: global_log.info('%s is already a parent of %s', parent.name, g.name) model.Session.commit() return g
def _create_member_request(context, data_dict): """ Helper to create member request """ changed = False role = data_dict['role'] group = model.Group.get(data_dict['group']) if not group or group.type != 'organization': raise NotFound user = context['user'] if new_authz.is_sysadmin(user): raise ValidationError({}, {_("Role"): _("As a sysadmin, you already have access to all organizations")}) userobj = model.User.get(user) member = model.Session.query(model.Member).filter(model.Member.table_name == "user").filter( model.Member.table_id == userobj.id) \ .filter(model.Member.group_id == group.id).filter( or_(model.Member.state == 'active', model.Member.state == 'pending')).first() if member: if member.state == 'pending': message = _("You already have a pending request to the organization") else: message = _("You are already part of the organization") raise ValidationError({"organization": _("Duplicate organization request")}, {_("Organization"): message}) member = model.Session.query(model.Member).filter(model.Member.table_name == "user").filter( model.Member.table_id == userobj.id) \ .filter(model.Member.group_id == group.id).first() if not member: member = model.Member(table_name="user", table_id=userobj.id, group_id=group.id, capacity=role, state='pending') changed = True locale = _get_safe_locale() if member.state != 'pending' or changed: member.state = 'pending' member.capacity = role revision = model.repo.new_revision() revision.author = user if 'message' in context: revision.message = context['message'] elif changed: revision.message = u'New member request' else: revision.message = u'Changed member request' if changed: model.Session.add(member) else: member.save() extra = MemberExtra(member_id=member.id, key="locale", value=locale) extra.save() model.repo.commit() changed = True url = config.get('ckan.site_url', "") if url: url = url + url_for('member_request_show', member_id=member.id) admin_list = [] if role == 'admin': for admin in get_ckan_admins(): admin_list.append({'display_name': admin.display_name, 'email': admin.email}) # _mail_new_membership_request(locale, admin, group, url, userobj, data_dict) else: for admin in get_organization_admins(group.id): admin_list.append({'display_name': admin.display_name, 'email': admin.email}) # _mail_new_membership_request(locale, admin, group, url, userobj, data_dict) _mail_new_membership_request(locale, None, group, url, userobj, data_dict, admin_list) return member, changed
def package_membership_list_save(group_dicts: Optional[list[dict[str, Any]]], package: 'model.Package', context: Context) -> None: allow_partial_update = context.get("allow_partial_update", False) if group_dicts is None and allow_partial_update: return capacity = 'public' model = context["model"] session = context["session"] user = context.get('user', '') members = session.query(model.Member) \ .filter(model.Member.table_id == package.id) \ .filter(model.Member.capacity != 'organization') group_member = dict((member.group, member) for member in members) groups: set[model.Group] = set() for group_dict in group_dicts or []: id = group_dict.get("id") name = group_dict.get("name") capacity = group_dict.get("capacity", "public") if capacity == 'organization': continue if id: group = session.query(model.Group).get(id) else: group = session.query(model.Group).filter_by(name=name).first() if group: groups.add(group) ## need to flush so we can get out the package id model.Session.flush() # Remove any groups we are no longer in for group in set(group_member.keys()) - groups: member_obj = group_member[group] if member_obj and member_obj.state == 'deleted': continue if authz.has_user_permission_for_group_or_org(member_obj.group_id, user, 'read'): member_obj.capacity = capacity member_obj.state = 'deleted' session.add(member_obj) # Add any new groups for group in groups: member_obj = group_member.get(group) if member_obj and member_obj.state == 'active': continue if authz.has_user_permission_for_group_or_org(group.id, user, 'read'): member_obj = group_member.get(group) if member_obj: member_obj.capacity = capacity member_obj.state = 'active' else: member_obj = model.Member(table_id=package.id, table_name='package', group=group, capacity=capacity, group_id=group.id, state='active') session.add(member_obj)
def nsw_package_membership_list_save(group_dicts, package, context): allow_partial_update = context.get("allow_partial_update", False) if group_dicts is None and allow_partial_update: return capacity = 'public' model = context["model"] session = context["session"] user = context.get('user') members = session.query(model.Member) \ .filter(model.Member.table_id == package.id) \ .filter(model.Member.capacity != 'organization') group_member = dict((member.group, member) for member in members) groups = set() for group_dict in group_dicts or []: id = group_dict.get("id") name = group_dict.get("name") capacity = group_dict.get("capacity", "public") if capacity == 'organization': continue if id: group = session.query(model.Group).get(id) else: group = session.query(model.Group).filter_by(name=name).first() if group: groups.add(group) ## need to flush so we can get out the package id model.Session.flush() # Remove any groups we are no longer in for group in set(group_member.keys()) - groups: member_obj = group_member[group] if member_obj and member_obj.state == 'deleted': continue # Bypass authorization to enable datasets to be removed from AGIFT classification member_obj.capacity = capacity member_obj.state = 'deleted' session.add(member_obj) # Add any new groups for group in groups: member_obj = group_member.get(group) if member_obj and member_obj.state == 'active': continue # Bypass authorization to enable datasets to be added to AGIFT classification member_obj = group_member.get(group) if member_obj: member_obj.capacity = capacity member_obj.state = 'active' else: member_obj = model.Member(table_id=package.id, table_name='package', group=group, capacity=capacity, group_id=group.id, state='active') session.add(member_obj)
def command(config_ini, nodepublisher_csv, users_csv): config_ini_filepath = os.path.abspath(config_ini) load_config(config_ini_filepath) engine = engine_from_config(config, 'sqlalchemy.') from ckan import model from ckan.lib.munge import munge_title_to_name import ckanext.dgu.plugin logging.config.fileConfig(config_ini_filepath) global log log = logging.getLogger(os.path.basename(__file__)) model.init_model(engine) # Register a translator in this thread so that # the _() functions in logic layer can work from ckan.lib.cli import MockTranslator registry = Registry() registry.prepare() translator_obj = MockTranslator() registry.register(translator, translator_obj) with open(nodepublisher_csv, 'rU') as f: reader = csv.reader(f) for row in reader: publishers[int(row[0])] = munge_title_to_name(row[1]) log.info('Opened list of %i publishers', reader.line_num) # get rid of flash message warnings warnings_.filterwarnings('ignore', '.*flash message.*') ckanext.dgu.plugin.log.disabled = True with open(users_csv, 'rU') as f: reader = csv.reader(f) for row in reader: model.repo.new_revision() node_id, name, email, publisher_id = row # create a new user uname, user_id = _add_new_user(int(node_id), name, email) if not user_id: # validation error. warning already printed continue # Find the publisher and add them as editor if node_id: publisher_name = publishers[int(publisher_id)] publisher = model.Group.by_name(publisher_name) if not publisher: warn( 'Could not find publisher %r so skipping making %r editor for it.', publisher_name, name) continue capacity = 'editor' # Check for Member where table_name is u['id'] res = model.Session.query(model.Member).\ from_statement(MEMBER_LOOKUP).\ params(userid=user_id, groupid=publisher.id).all() if len(res) == 0: m = model.Member(group_id=publisher.id, table_id=user_id, table_name='user', capacity=capacity) model.Session.add(m) log.info('Made %r editor for %r', name, publisher_name) else: log.info('%r already editor for %r', name, publisher_name) # Update harvest_source user_id field to new user id. model.Session.execute(HARVEST_QUERY, params={ 'uid': user_id, 'node_id': str(node_id) }) model.Session.commit() log.info('Processed list of %i users', reader.line_num) log.info('Warnings (%i): %r', len(warnings), warnings)