def user_dictize(user, context): if context.get('with_capacity'): user, capacity = user result_dict = d.table_dictize(user, context, capacity=capacity) else: result_dict = d.table_dictize(user, context) del result_dict['password'] result_dict['display_name'] = user.display_name result_dict['email_hash'] = user.email_hash result_dict['number_of_edits'] = user.number_of_edits() result_dict['number_administered_packages'] = user.number_administered_packages() model = context['model'] session = model.Session if context.get('with_related'): related_items = session.query(model.Related).\ filter(model.Related.owner_id==user.id).all() result_dict['related_items'] = related_list_dictize(related_items, context) return result_dict
def group_list_dictize(obj_list, context, sort_key=lambda x:x['display_name'], reverse=False): active = context.get('active', True) with_private = context.get('include_private_packages', False) result_list = [] for obj in obj_list: if context.get('with_capacity'): obj, capacity = obj group_dict = d.table_dictize(obj, context, capacity=capacity) else: group_dict = d.table_dictize(obj, context) group_dict.pop('created') if active and obj.state not in ('active', 'pending'): continue group_dict['display_name'] = obj.display_name group_dict['packages'] = \ len(obj.active_packages(with_private=with_private).all()) if context.get('for_view'): for item in plugins.PluginImplementations( plugins.IGroupController): group_dict = item.before_view(group_dict) result_list.append(group_dict) return sorted(result_list, key=sort_key, reverse=reverse)
def user_dictize(user, context): if context.get('with_capacity'): user, capacity = user result_dict = d.table_dictize(user, context, capacity=capacity) else: result_dict = d.table_dictize(user, context) del result_dict['password'] result_dict['display_name'] = user.display_name result_dict['email_hash'] = user.email_hash result_dict['number_of_edits'] = user.number_of_edits() result_dict['number_administered_packages'] = user.number_administered_packages() requester = context.get('user') if not (new_authz.is_sysadmin(requester) or requester == user.name or context.get('keep_sensitive_data', False)): # If not sysadmin or the same user, strip sensible info result_dict.pop('apikey', None) result_dict.pop('reset_key', None) result_dict.pop('email', None) model = context['model'] session = model.Session if context.get('with_related'): related_items = session.query(model.Related).\ filter(model.Related.owner_id==user.id).all() result_dict['related_items'] = related_list_dictize(related_items, context) return result_dict
def tag_list_dictize(tag_list, context): result_list = [] for tag in tag_list: if context.get('with_capacity'): tag, capacity = tag dictized = d.table_dictize(tag, context, capacity=capacity) else: dictized = d.table_dictize(tag, context) # Add display_names to tag dicts. At first a tag's display_name is just # the same as its name, but the display_name might get changed later # (e.g. translated into another language by the multilingual # extension). assert not dictized.has_key('display_name') dictized['display_name'] = dictized['name'] if context.get('for_view'): for item in plugins.PluginImplementations( plugins.ITagController): dictized = item.before_view(dictized) result_list.append(dictized) return result_list
def test_01_dictize_main_objects_simple(self): context = {"model": model, "session": model.Session} ## package pkg = model.Session.query(model.Package).filter_by(name="annakarenina").first() result = table_dictize(pkg, context) self.remove_changable_columns(result) assert result == { "author": None, "author_email": None, "license_id": u"other-open", "maintainer": None, "maintainer_email": None, "name": u"annakarenina", "notes": u"Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n", "state": u"active", "title": u"A Novel By Tolstoy", "url": u"http://www.annakarenina.com", "version": u"0.7a", }, pprint(result) ## resource resource = pkg.resource_groups[0].resources[0] result = resource_dictize(resource, context) self.remove_changable_columns(result) assert result == { u"alt_url": u"alt123", "cache_last_updated": None, "cache_url": None, "description": u'Full text. Needs escaping: " Umlaut: \xfc', "format": u"plain text", "hash": u"abc123", "last_modified": None, "mimetype": None, "mimetype_inner": None, "name": None, "position": 0, "resource_type": None, "size": None, u"size_extra": u"123", "state": u"active", "url": u"http://www.annakarenina.com/download/x=1&y=2", "webstore_last_updated": None, "webstore_url": None, }, pprint(result) ## package extra key, package_extras = pkg._extras.popitem() result = table_dictize(package_extras, context) self.remove_changable_columns(result) assert result == {"key": u"genre", "state": u"active", "value": u"romantic novel"}, pprint(result)
def group_list_dictize(obj_list, context, sort_key=lambda x:x['display_name'], reverse=False, with_package_counts=True): active = context.get('active', True) with_private = context.get('include_private_packages', False) if with_package_counts: query = search.PackageSearchQuery() q = {'q': '+capacity:public' if not with_private else '*:*', 'fl': 'groups', 'facet.field': ['groups', 'owner_org'], 'facet.limit': -1, 'rows': 1} query.run(q) result_list = [] for obj in obj_list: if context.get('with_capacity'): obj, capacity = obj group_dict = d.table_dictize(obj, context, capacity=capacity) else: group_dict = d.table_dictize(obj, context) group_dict.pop('created') if active and obj.state not in ('active', 'pending'): continue group_dict['display_name'] = (group_dict.get('title') or group_dict.get('name')) image_url = group_dict.get('image_url') group_dict['image_display_url'] = image_url if image_url and not image_url.startswith('http'): #munge here should not have an effect only doing it incase #of potential vulnerability of dodgy api input image_url = munge.munge_filename(image_url) group_dict['image_display_url'] = h.url_for_static( 'uploads/group/%s' % group_dict.get('image_url'), qualified=True ) if with_package_counts: facets = query.facets if obj.is_organization: group_dict['packages'] = facets['owner_org'].get(obj.id, 0) else: group_dict['packages'] = facets['groups'].get(obj.name, 0) if context.get('for_view'): if group_dict['is_organization']: plugin = plugins.IOrganizationController else: plugin = plugins.IGroupController for item in plugins.PluginImplementations(plugin): group_dict = item.before_view(group_dict) result_list.append(group_dict) return sorted(result_list, key=sort_key, reverse=reverse)
def test_01_dictize_main_objects_simple(self): context = {"model": model, "session": model.Session} ## package pkg = model.Session.query(model.Package).filter_by(name='annakarenina').first() result = table_dictize(pkg, context) self.remove_changable_columns(result) assert result == { 'author': None, 'author_email': None, 'license_id': u'other-open', 'maintainer': None, 'maintainer_email': None, 'name': u'annakarenina', 'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n', 'state': u'active', 'title': u'A Novel By Tolstoy', 'url': u'http://www.annakarenina.com', 'version': u'0.7a' }, pprint(result) ## resource resource = pkg.resource_groups[0].resources[0] result = resource_dictize(resource, context) self.remove_changable_columns(result) assert result == { 'alt_url': u'alt123', 'description': u'Full text. Needs escaping: " Umlaut: \xfc', 'format': u'plain text', 'hash': u'abc123', 'position': 0, 'size': u'123', 'state': u'active', 'url': u'http://www.annakarenina.com/download/x=1&y=2'}, pprint(result) ## package extra key, package_extras = pkg._extras.popitem() result = table_dictize(package_extras, context) self.remove_changable_columns(result) assert result == { 'key': u'genre', 'state': u'active', 'value': u'romantic novel' }, pprint(result)
def tag_list_dictize(tag_list, context): result_list = [] for tag in tag_list: if context.get('with_capacity'): tag, capacity = tag dictized = d.table_dictize(tag, context, capacity=capacity) else: dictized = d.table_dictize(tag, context) result_list.append(dictized) return result_list
def user_dictize(user, context, include_password_hash=False): if context.get('with_capacity'): user, capacity = user result_dict = d.table_dictize(user, context, capacity=capacity) else: result_dict = d.table_dictize(user, context) password_hash = result_dict.pop('password') del result_dict['reset_key'] result_dict['display_name'] = user.display_name result_dict['email_hash'] = user.email_hash result_dict['number_of_edits'] = user.number_of_edits() result_dict['number_created_packages'] = user.number_created_packages( include_private_and_draft=context.get( 'count_private_and_draft_datasets', False)) requester = context.get('user') reset_key = result_dict.pop('reset_key', None) apikey = result_dict.pop('apikey', None) email = result_dict.pop('email', None) if context.get('keep_email', False): result_dict['email'] = email if context.get('keep_apikey', False): result_dict['apikey'] = apikey if requester == user.name: result_dict['apikey'] = apikey result_dict['email'] = email if authz.is_sysadmin(requester): result_dict['apikey'] = apikey result_dict['email'] = email if include_password_hash: result_dict['password_hash'] = password_hash model = context['model'] session = model.Session if context.get('with_related'): related_items = session.query(model.Related).\ filter(model.Related.owner_id==user.id).all() result_dict['related_items'] = related_list_dictize(related_items, context) return result_dict
def revision_to_activity(r): pr = packages_previous.get(r.revision_id) if only_privatized and (pr is None or (pr.private or not r.private)): return None privacy_changed = pr is not None and pr.private != r.private activity_type = None if r.state in ('active', 'draft'): if packages_created[r.id] == r.revision_timestamp: activity_type = 'new package' elif privacy_changed: activity_type = 'changed package privacy' else: activity_type = 'changed package' elif r.state in ('deleted'): activity_type = 'deleted package' else: log.warning("Unknown package state, skipping: %s" % r.state) return None d = {'package': dictization.table_dictize( packages[r.id], context={'model': model})} activity = model.Activity(r.creator_user_id, r.id, r.revision_id, activity_type, d) activity.timestamp = r.revision_timestamp return activity
def vocabulary_dictize(vocabulary, context, include_datasets=False): vocabulary_dict = d.table_dictize(vocabulary, context) assert not vocabulary_dict.has_key('tags') vocabulary_dict['tags'] = [tag_dictize(tag, context, include_datasets) for tag in vocabulary.tags] return vocabulary_dict
def current_package_list_with_resources(context, data_dict): model = context["model"] user = context["user"] limit = data_dict.get("limit") q = ckan.authz.Authorizer().authorized_query(user, model.PackageRevision) q = q.filter(model.PackageRevision.state=='active') q = q.filter(model.PackageRevision.current==True) q = q.order_by(model.package_revision_table.c.revision_timestamp.desc()) if limit: q = q.limit(limit) pack_rev = q.all() package_list = [] for package in pack_rev: result_dict = table_dictize(package, context) res_rev = model.resource_revision_table resource_group = model.resource_group_table q = select([res_rev], from_obj = res_rev.join(resource_group, resource_group.c.id == res_rev.c.resource_group_id)) q = q.where(resource_group.c.package_id == package.id) result = q.where(res_rev.c.current == True).execute() result_dict["resources"] = resource_list_dictize(result, context) license_id = result_dict['license_id'] if license_id: isopen = model.Package.get_license_register()[license_id].isopen() result_dict['isopen'] = isopen else: result_dict['isopen'] = False package_list.append(result_dict) return package_list
def resource_dictize(res, context): resource = d.table_dictize(res, context) resource['format'] = resource.get('format').lower() if resource.get('format') else '' extras = resource.pop("extras", None) if extras: resource.update(extras) return resource
def group_dictize(group, context): model = context["model"] result_dict = table_dictize(group, context) result_dict["display_name"] = group.display_name result_dict["extras"] = extras_dict_dictize(group._extras, context) context["with_capacity"] = True result_dict["packages"] = obj_list_dictize(_get_members(context, group, "packages"), context) result_dict["tags"] = tag_list_dictize(_get_members(context, group, "tags"), context) result_dict["groups"] = group_list_dictize(_get_members(context, group, "groups"), context) result_dict["users"] = user_list_dictize(_get_members(context, group, "users"), context) context["with_capacity"] = False if context.get("for_view"): for item in PluginImplementations(IGroupController): result_dict = item.before_view(result_dict) return result_dict
def inventory_entry_bulk_create(context, data_dict): model = context['model'] schema = context['schema'] session = context['session'] organization = model.Group.get(context['organization_name']) inventory_entry_dict = {'group_id': organization.id} if not data_dict['field-name-input-0']: raise ValidationError({'error': [_('Please add at least one inventory entry.')]}) for inventory_entry_id in range(10): inventory_entry_name = data_dict['field-name-input-' + str(inventory_entry_id)] if not inventory_entry_name: break inventory_entry_dict['title'] = inventory_entry_name inventory_entry_dict['recurring_interval'] = data_dict['field-recurring-input-' + str(inventory_entry_id)] inventory_entry_dict['is_recurring'] = (inventory_entry_dict['recurring_interval'] != '0') data, errors = navl_validate(inventory_entry_dict, schema, context) if errors: session.rollback() # TODO @palcu: i18n raise ValidationError({'error': [_('Verificati intrarea cu numarul {0}.'.format(inventory_entry_id+1))]}) obj = table_dict_save(inventory_entry_dict, InventoryEntry, context) model.repo.commit() return table_dictize(obj, context)
def resource_dictize(res, context): model = context['model'] resource = d.table_dictize(res, context) resource_group_id = resource['resource_group_id'] extras = resource.pop("extras", None) if extras: resource.update(extras) # some urls do not have the protocol this adds http:// to these url = resource['url'] ## for_edit is only called at the times when the dataset is to be edited ## in the frontend. Without for_edit the whole qualified url is returned. if resource.get('url_type') == 'upload' and not context.get('for_edit'): resource_group = model.Session.query( model.ResourceGroup).get(resource_group_id) last_part = url.split('/')[-1] cleaned_name = munge.munge_filename(last_part) resource['url'] = h.url_for(controller='package', action='resource_download', id=resource_group.package_id, resource_id=res.id, filename=cleaned_name, qualified=True) elif not urlparse.urlsplit(url).scheme and not context.get('for_edit'): resource['url'] = u'http://' + url.lstrip('/') return resource
def inventory_entry_list(context, data_dict): '''Return a list of inventory entries. :param name: organization name :type name: string :rtype: list of dictionaries ''' # TODO @palcu: define this # check_access('inventory_manage', context, data_dict) model = context['model'] name = get_or_bust(data_dict, 'name') organization = model.Group.get(name) if not organization: raise ObjectNotFound('Organization was not found') entries = [ table_dictize(entry, context) for entry in organization.inventory_entries] for entry in entries: entry['next_deadline_timestamp'] = None if entry['last_added_dataset_timestamp']: last_added = _datestamp_to_datetime(entry['last_added_dataset_timestamp']) delta = timedelta(days=entry['recurring_interval']) entry['next_deadline_timestamp'] = last_added + delta return entries
def user_dictize(user, context): if context.get("with_capacity"): user, capacity = user result_dict = table_dictize(user, context, capacity=capacity) else: result_dict = table_dictize(user, context) del result_dict["password"] result_dict["display_name"] = user.display_name result_dict["email_hash"] = user.email_hash result_dict["number_of_edits"] = user.number_of_edits() result_dict["number_administered_packages"] = user.number_administered_packages() return result_dict
def tag_dictize(tag, context): result_dict = table_dictize(tag, context) result_dict["packages"] = obj_list_dictize(tag.packages, context) return result_dict
def civicdata_featured_upsert(context, data_dict): data, errors = df.validate(data_dict, schema, context) if errors: raise ValidationError(errors) featured = db.Civicdata_Featured.get(resource_view_id=data['resource_view_id']) if featured is None: featured = db.Civicdata_Featured() featured.resource_view_id = data['resource_view_id'] if data.has_key('canonical'): featured.canonical = data['canonical'] if data.has_key('organizationpage'): featured.organizationpage = data['organizationpage'] resource_id = model.ResourceView.get(featured.resource_view_id).resource_id featured.package_id = model.Resource.get(resource_id).package_id featured.save() session = context['session'] session.add(featured) session.commit() return table_dictize(featured, context)
def _package_list_with_resources(context, package_revision_list): package_list = [] model = context["model"] for package in package_revision_list: result_dict = table_dictize(package, context) res_rev = model.resource_revision_table resource_group = model.resource_group_table query = select( [res_rev], from_obj=res_rev.join(resource_group, resource_group.c.id == res_rev.c.resource_group_id) ) query = query.where(resource_group.c.package_id == package.id) result = query.where(res_rev.c.current == True).execute() result_dict["resources"] = resource_list_dictize(result, context) license_id = result_dict["license_id"] if license_id: try: isopen = model.Package.get_license_register()[license_id].isopen() result_dict["isopen"] = isopen except KeyError: # TODO: create a log message this error? result_dict["isopen"] = False else: result_dict["isopen"] = False package_list.append(result_dict) return package_list
def tag_dictize(tag, context): tag_dict = d.table_dictize(tag, context) query = search.PackageSearchQuery() q = {'q': '+tags:"%s" +capacity:public' % tag.name, 'fl': 'data_dict', 'wt': 'json', 'rows': 1000} package_dicts = [h.json.loads(result['data_dict']) for result in query.run(q)['results']] # Add display_names to tags. At first a tag's display_name is just the # same as its name, but the display_name might get changed later (e.g. # translated into another language by the multilingual extension). assert not tag_dict.has_key('display_name') tag_dict['display_name'] = tag_dict['name'] if context.get('for_view'): for item in plugins.PluginImplementations(plugins.ITagController): tag_dict = item.before_view(tag_dict) tag_dict['packages'] = [] for package_dict in package_dicts: for item in plugins.PluginImplementations(plugins.IPackageController): package_dict = item.before_view(package_dict) tag_dict['packages'].append(package_dict) else: tag_dict['packages'] = package_dicts return tag_dict
def group_dictize(group, context): model = context['model'] result_dict = d.table_dictize(group, context) result_dict['display_name'] = group.display_name result_dict['extras'] = extras_dict_dictize( group._extras, context) context['with_capacity'] = True result_dict['packages'] = d.obj_list_dictize( _get_members(context, group, 'packages'), context) result_dict['tags'] = tag_list_dictize( _get_members(context, group, 'tags'), context) result_dict['groups'] = group_list_dictize( _get_members(context, group, 'groups'), context) result_dict['users'] = user_list_dictize( _get_members(context, group, 'users'), context) context['with_capacity'] = False if context.get('for_view'): for item in plugins.PluginImplementations(plugins.IGroupController): result_dict = item.before_view(result_dict) return result_dict
def user_dictize(user, context): if context.get('with_capacity'): user, capacity = user result_dict = d.table_dictize(user, context, capacity=capacity) else: result_dict = d.table_dictize(user, context) del result_dict['password'] result_dict['display_name'] = user.display_name result_dict['email_hash'] = user.email_hash result_dict['number_of_edits'] = user.number_of_edits() result_dict['number_administered_packages'] = user.number_administered_packages() return result_dict
def group_dictize(group, context): model = context['model'] result_dict = table_dictize(group, context) result_dict['display_name'] = group.display_name result_dict['extras'] = extras_dict_dictize( group._extras, context) context['with_capacity'] = True result_dict['packages'] = obj_list_dictize( _get_members(context, group, 'packages'), context) result_dict['tags'] = tag_list_dictize( _get_members(context, group, 'tags'), context) result_dict['groups'] = group_list_dictize( _get_members(context, group, 'groups'), context) result_dict['users'] = user_list_dictize( _get_members(context, group, 'users'), context) context['with_capacity'] = False return result_dict
def activity_stream_item(self, activity_type, revision, user_id): import ckan.model import ckan.logic assert activity_type in ("new", "changed"), ( str(activity_type)) # Handle 'deleted' objects. # When the user marks a package as deleted this comes through here as # a 'changed' package activity. We detect this and change it to a # 'deleted' activity. if activity_type == 'changed' and self.state == u'deleted': if meta.Session.query(activity.Activity).filter_by( object_id=self.id, activity_type='deleted').all(): # A 'deleted' activity for this object has already been emitted # FIXME: What if the object was deleted and then activated # again? return None else: # Emit a 'deleted' activity for this object. activity_type = 'deleted' try: d = {'package': dictization.table_dictize(self, context={'model': ckan.model})} return activity.Activity(user_id, self.id, revision.id, "%s package" % activity_type, d) except ckan.logic.NotFound: # This happens if this package is being purged and therefore has no # current revision. # TODO: Purge all related activity stream items when a model object # is purged. return None
def user_dictize(user, context): if context.get('with_capacity'): user, capacity = user result_dict = d.table_dictize(user, context, capacity=capacity) else: result_dict = d.table_dictize(user, context) del result_dict['password'] del result_dict['reset_key'] result_dict['display_name'] = user.display_name result_dict['email_hash'] = user.email_hash result_dict['number_of_edits'] = user.number_of_edits() result_dict['number_administered_packages'] = user.number_administered_packages() requester = context.get('user') reset_key = result_dict.pop('reset_key', None) apikey = result_dict.pop('apikey', None) email = result_dict.pop('email', None) if context.get('keep_email', False): result_dict['email'] = email if context.get('keep_apikey', False): result_dict['apikey'] = apikey if requester == user.name: result_dict['apikey'] = apikey result_dict['email'] = email ## this should not really really be needed but tests need it if new_authz.is_sysadmin(requester): result_dict['apikey'] = apikey result_dict['email'] = email model = context['model'] session = model.Session if context.get('with_related'): related_items = session.query(model.Related).\ filter(model.Related.owner_id==user.id).all() result_dict['related_items'] = related_list_dictize(related_items, context) return result_dict
def group_dictize(group, context): model = context['model'] result_dict = d.table_dictize(group, context) result_dict['display_name'] = group.display_name result_dict['extras'] = extras_dict_dictize( group._extras, context) context['with_capacity'] = True result_dict['packages'] = d.obj_list_dictize( _get_members(context, group, 'packages'), context) query = search.PackageSearchQuery() if group.is_organization: q = {'q': 'owner_org:"%s" +capacity:public' % group.id, 'rows': 1} else: q = {'q': 'groups:"%s" +capacity:public' % group.name, 'rows': 1} package_count = query.run(q)['count'] result_dict['package_count'] = package_count # if group.is_organization: # q = {'q': 'owner_org:"%s" +capacity:public' % group.id, 'rows': package_count} # packages = query.run(q)['results'] ## log.error(str(packages)) # result_dict['packages'] = packages result_dict['tags'] = tag_list_dictize( _get_members(context, group, 'tags'), context) result_dict['groups'] = group_list_dictize( _get_members(context, group, 'groups'), context) if context.get('for_view'): context['user'] = '******' result_dict['users'] = user_list_dictize( _get_members(context, group, 'users'), context) context['with_capacity'] = False if context.get('for_view'): if result_dict['is_organization']: plugin = plugins.IOrganizationController else: plugin = plugins.IGroupController for item in plugins.PluginImplementations(plugin): result_dict = item.before_view(result_dict) return result_dict
def test_07_table_simple_save(self): context = {"model": model, "session": model.Session} anna1 = model.Session.query(model.Package).filter_by(name='annakarenina').one() anna_dictized = self.remove_changable_columns(table_dictize(anna1, context)) anna_dictized["name"] = 'annakarenina2' model.repo.new_revision() table_dict_save(anna_dictized, model.Package, context) model.Session.commit() pkg = model.Session.query(model.Package).filter_by(name='annakarenina2').one() assert self.remove_changable_columns(table_dictize(pkg, context)) == anna_dictized, self.remove_changable_columns(table_dictize(pkg, context))
def resource_view_dictize(resource_view, context): dictized = d.table_dictize(resource_view, context) dictized.pop('order') config = dictized.pop('config', {}) dictized.update(config) resource = context['model'].Resource.get(resource_view.resource_id) package_id = resource.package_id dictized['package_id'] = package_id return dictized
def as_dict(self): context = {'model': model} rating_dict = dictization.table_dictize(self, context) return rating_dict
def _group_or_org_update(context, data_dict, is_org=False): model = context['model'] user = context['user'] session = context['session'] id = _get_or_bust(data_dict, 'id') group = model.Group.get(id) context["group"] = group if group is None: raise NotFound('Group was not found.') data_dict['type'] = group.type # get the schema group_plugin = lib_plugins.lookup_group_plugin(group.type) try: schema = group_plugin.form_to_db_schema_options({ 'type': 'update', 'api': 'api_version' in context, 'context': context }) except AttributeError: schema = group_plugin.form_to_db_schema() upload = uploader.get_uploader('group', group.image_url) upload.update_data_dict(data_dict, 'image_url', 'image_upload', 'clear_upload') if is_org: _check_access('organization_update', context, data_dict) else: _check_access('group_update', context, data_dict) if 'api_version' not in context: # old plugins do not support passing the schema so we need # to ensure they still work try: group_plugin.check_data_dict(data_dict, schema) except TypeError: group_plugin.check_data_dict(data_dict) data, errors = lib_plugins.plugin_validate( group_plugin, context, data_dict, schema, 'organization_update' if is_org else 'group_update') log.debug('group_update validate_errs=%r user=%s group=%s data_dict=%r', errors, context.get('user'), context.get('group').name if context.get('group') else '', data_dict) if errors: session.rollback() raise ValidationError(errors) rev = model.repo.new_revision() rev.author = user if 'message' in context: rev.message = context['message'] else: rev.message = _(u'REST API: Update object %s') % data.get("name") group = model_save.group_dict_save(data, context, prevent_packages_update=is_org) if is_org: plugin_type = plugins.IOrganizationController else: plugin_type = plugins.IGroupController for item in plugins.PluginImplementations(plugin_type): item.edit(group) if is_org: activity_type = 'changed organization' else: activity_type = 'changed group' activity_dict = { 'user_id': model.User.by_name(user.decode('utf8')).id, 'object_id': group.id, 'activity_type': activity_type, } # Handle 'deleted' groups. # When the user marks a group as deleted this comes through here as # a 'changed' group activity. We detect this and change it to a 'deleted' # activity. if group.state == u'deleted': if session.query(ckan.model.Activity).filter_by( object_id=group.id, activity_type='deleted').all(): # A 'deleted group' activity for this group has already been # emitted. # FIXME: What if the group was deleted and then activated again? activity_dict = None else: # We will emit a 'deleted group' activity. activity_dict['activity_type'] = 'deleted group' if activity_dict is not None: activity_dict['data'] = { 'group': dictization.table_dictize(group, context) } activity_create_context = { 'model': model, 'user': user, 'defer_commit': True, 'ignore_auth': True, 'session': session } _get_action('activity_create')(activity_create_context, activity_dict) # TODO: Also create an activity detail recording what exactly changed # in the group. upload.upload(uploader.get_max_image_size()) if not context.get('defer_commit'): model.repo.commit() return model_dictize.group_dictize(group, context)
def vocabulary_dictize(vocabulary, context): vocabulary_dict = d.table_dictize(vocabulary, context) assert not vocabulary_dict.has_key('tags') vocabulary_dict['tags'] = [tag_dictize(tag, context) for tag in vocabulary.tags] return vocabulary_dict
def resource_dictize(res, context): resource = d.table_dictize(res, context) extras = resource.pop("extras", None) if extras: resource.update(extras) return resource
def for_json(self, context): return table_dictize(self, context)
def user_following_group_dictize(follower, context): return d.table_dictize(follower, context)
def _group_or_org_update(context, data_dict, is_org=False): model = context['model'] user = context['user'] session = context['session'] id = _get_or_bust(data_dict, 'id') parent = context.get('parent', None) group = model.Group.get(id) context["group"] = group if group is None: raise NotFound('Group was not found.') # get the schema group_plugin = lib_plugins.lookup_group_plugin(group.type) try: schema = group_plugin.form_to_db_schema_options({ 'type': 'update', 'api': 'api_version' in context, 'context': context }) except AttributeError: schema = group_plugin.form_to_db_schema() if is_org: _check_access('organization_update', context, data_dict) else: _check_access('group_update', context, data_dict) if 'api_version' not in context: # old plugins do not support passing the schema so we need # to ensure they still work try: group_plugin.check_data_dict(data_dict, schema) except TypeError: group_plugin.check_data_dict(data_dict) data, errors = _validate(data_dict, schema, context) log.debug('group_update validate_errs=%r user=%s group=%s data_dict=%r', errors, context.get('user'), context.get('group').name if context.get('group') else '', data_dict) if errors: session.rollback() raise ValidationError(errors) rev = model.repo.new_revision() rev.author = user if 'message' in context: rev.message = context['message'] else: rev.message = _(u'REST API: Update object %s') % data.get("name") # when editing an org we do not want to update the packages if using the # new templates. if ((not is_org) and not converters.asbool(config.get('ckan.legacy_templates', False)) and 'api_version' not in context): context['prevent_packages_update'] = True group = model_save.group_dict_save(data, context) if parent: parent_group = model.Group.get(parent) if parent_group and not parent_group in group.get_groups(group.type): # Delete all of this groups memberships current = session.query(model.Member).\ filter(model.Member.table_id == group.id).\ filter(model.Member.table_name == "group").all() if current: log.debug('Parents of group %s deleted: %r', group.name, [membership.group.name for membership in current]) for c in current: session.delete(c) member = model.Member(group=parent_group, table_id=group.id, table_name='group') session.add(member) log.debug('Group %s is made child of group %s', group.name, parent_group.name) if is_org: plugin_type = plugins.IOrganizationController else: plugin_type = plugins.IGroupController for item in plugins.PluginImplementations(plugin_type): item.edit(group) if is_org: activity_type = 'changed organization' else: activity_type = 'changed group' activity_dict = { 'user_id': model.User.by_name(user.decode('utf8')).id, 'object_id': group.id, 'activity_type': activity_type, } # Handle 'deleted' groups. # When the user marks a group as deleted this comes through here as # a 'changed' group activity. We detect this and change it to a 'deleted' # activity. if group.state == u'deleted': if session.query(ckan.model.Activity).filter_by( object_id=group.id, activity_type='deleted').all(): # A 'deleted group' activity for this group has already been # emitted. # FIXME: What if the group was deleted and then activated again? activity_dict = None else: # We will emit a 'deleted group' activity. activity_dict['activity_type'] = 'deleted group' if activity_dict is not None: activity_dict['data'] = { 'group': dictization.table_dictize(group, context) } activity_create_context = { 'model': model, 'user': user, 'defer_commit': True, 'session': session } _get_action('activity_create')(activity_create_context, activity_dict, ignore_auth=True) # TODO: Also create an activity detail recording what exactly changed # in the group. if not context.get('defer_commit'): model.repo.commit() return model_dictize.group_dictize(group, context)
def tag_dictize(tag, context): result_dict = d.table_dictize(tag, context) result_dict["packages"] = d.obj_list_dictize(tag.packages, context) return result_dict
def group_dictize(group, context, include_groups=True, include_tags=True, include_users=True, include_extras=True, packages_field='datasets', **kw): ''' Turns a Group object and related into a dictionary. The related objects like tags are included unless you specify it in the params. :param packages_field: determines the format of the `packages` field - can be `datasets`, `dataset_count`, `none_but_include_package_count` or None. If set to `dataset_count` or `none_but_include_package_count` then you can precalculate dataset counts in advance by supplying: context['dataset_counts'] = get_group_dataset_counts() ''' assert packages_field in ('datasets', 'dataset_count', 'none_but_include_package_count', None) if packages_field in ('dataset_count', 'none_but_include_package_count'): dataset_counts = context.get('dataset_counts', None) result_dict = d.table_dictize(group, context) result_dict.update(kw) result_dict['display_name'] = group.title or group.name if include_extras: result_dict['extras'] = extras_dict_dictize( group._extras, context) context['with_capacity'] = True if packages_field: def get_packages_for_this_group(group_): # Ask SOLR for the list of packages for this org/group q = { 'facet': 'false', 'rows': 0, } if group_.is_organization: q['fq'] = 'owner_org:"{0}"'.format(group_.id) else: q['fq'] = 'groups:"{0}"'.format(group_.name) is_group_member = (context.get('user') and new_authz.has_user_permission_for_group_or_org(group_.id, context.get('user'), 'read')) if is_group_member: context['ignore_capacity_check'] = True if not context.get('for_view'): q['rows'] = 1000 # Only the first 1000 datasets are returned search_context = dict((k, v) for (k, v) in context.items() if k != 'schema') search_results = logic.get_action('package_search')(search_context, q) return search_results['count'], search_results['results'] if packages_field == 'datasets': package_count, packages = get_packages_for_this_group(group) result_dict['packages'] = packages else: # i.e. packages_field is 'dataset_count' or # 'none_but_include_package_count' if dataset_counts is None: package_count, packages = get_packages_for_this_group(group) else: # Use the pre-calculated package_counts passed in. facets = dataset_counts if group.is_organization: package_count = facets['owner_org'].get(group.id, 0) else: package_count = facets['groups'].get(group.name, 0) if packages_field != 'none_but_include_package_count': result_dict['packages'] = package_count result_dict['package_count'] = package_count if include_tags: # group tags are not creatable via the API yet, but that was(/is) a # future intention (see kindly's commit 5c8df894 on 2011/12/23) result_dict['tags'] = tag_list_dictize( _get_members(context, group, 'tags'), context) if include_groups: # these sub-groups won't have tags or extras for speed result_dict['groups'] = group_list_dictize( _get_members(context, group, 'groups'), context, include_groups=True) if include_users: result_dict['users'] = user_list_dictize( _get_members(context, group, 'users'), context) context['with_capacity'] = False if context.get('for_view'): if result_dict['is_organization']: plugin = plugins.IOrganizationController else: plugin = plugins.IGroupController for item in plugins.PluginImplementations(plugin): result_dict = item.before_view(result_dict) image_url = result_dict.get('image_url') result_dict['image_display_url'] = image_url if image_url and not image_url.startswith('http'): #munge here should not have an effect only doing it incase #of potential vulnerability of dodgy api input image_url = munge.munge_filename(image_url) result_dict['image_display_url'] = h.url_for_static( 'uploads/group/%s' % result_dict.get('image_url'), qualified = True ) return result_dict
def member_dictize(member, context): return d.table_dictize(member, context)
def notify_maintainer(pkg_dict, maintainer): if not is_maintainer_notification(): return object_id_validators[activity_assign_maintainer] = tk.get_validator( 'user_id_exists') context = { 'model': model, 'session': ckan.model.Session, 'user': tk.c.user or tk.c.author, 'ignore_auth': True } admin_user = tk.get_action('get_site_user')(context, {}) dataset_context = { 'model': model, 'session': ckan.model.Session, 'user': tk.c.user or tk.c.author, 'api_version': 3, 'for_edit': True, 'auth_user_obj': admin_user, 'ignore_auth': True } activity_data = {} activity_object_id = pkg_dict.get('id') if not activity_object_id: return #modification check package = model.Package.get(activity_object_id) package_revision = model.package.package_revision_table cols = [ package_revision.c.maintainer, package_revision.c.metadata_modified ] sql = select(cols, from_obj=[ package_revision ]).where(package_revision.c.id == activity_object_id).order_by( package_revision.c.revision_timestamp.desc()) rows = model.Session.execute(sql).fetchall() if rows[0]['metadata_modified'] != package.metadata_modified: return #no updated content in package table, including maintainer if len(rows) > 1 and rows[1]['maintainer'] == maintainer.name: return #if maintainer is the same as before #retrieve package data as activity_data try: activity_data = tk.get_action('package_show')(dataset_context, { 'id': activity_object_id }) except NotFound: #at new creation of the package, the data has not been existing in the DB, so retrieve the data from pkg_dict and sanitize it activity_data = copy.deepcopy(pkg_dict) _sanitize_dict(activity_data) #sanitize maintainer_dict = table_dictize(maintainer, context) #add item into the creators activity stream indicating the maintainer has been assigned activity_dict = { 'user_id': tk.c.user or tk.c.author, 'object_id': maintainer.id, 'data': { 'dataset': activity_data, 'user': maintainer_dict }, 'activity_type': activity_assign_maintainer, } tk.get_action('activity_create')(context, activity_dict) print 'done'
def user_dictize(user, context, include_password_hash=False, include_plugin_extras=False): if context.get('with_capacity'): user, capacity = user result_dict = d.table_dictize(user, context, capacity=capacity) else: result_dict = d.table_dictize(user, context) password_hash = result_dict.pop('password') del result_dict['reset_key'] result_dict['display_name'] = user.display_name result_dict['email_hash'] = user.email_hash result_dict['number_created_packages'] = user.number_created_packages( include_private_and_draft=context.get( 'count_private_and_draft_datasets', False)) requester = context.get('user') reset_key = result_dict.pop('reset_key', None) apikey = result_dict.pop('apikey', None) email = result_dict.pop('email', None) plugin_extras = result_dict.pop('plugin_extras', None) if context.get('keep_email', False): result_dict['email'] = email if context.get('keep_apikey', False): result_dict['apikey'] = apikey if requester == user.name: result_dict['apikey'] = apikey result_dict['email'] = email if authz.is_sysadmin(requester): result_dict['apikey'] = apikey result_dict['email'] = email if include_password_hash: result_dict['password_hash'] = password_hash if include_plugin_extras: result_dict['plugin_extras'] = copy.deepcopy( plugin_extras) if plugin_extras else plugin_extras model = context['model'] session = model.Session image_url = result_dict.get('image_url') result_dict['image_display_url'] = image_url if image_url and not image_url.startswith('http'): # munge here should not have any effect, only doing it in case # of potential vulnerability of dodgy api input. image_url = munge.munge_filename_legacy(image_url) result_dict['image_display_url'] = h.url_for_static( 'uploads/user/%s' % result_dict.get('image_url'), qualified=True) return result_dict
def group_dictize(group, context, include_groups=True, include_tags=True, include_users=True, include_extras=True, packages_field='datasets', **kw): ''' Turns a Group object and related into a dictionary. The related objects like tags are included unless you specify it in the params. :param packages_field: determines the format of the `packages` field - can be `datasets`, `dataset_count` or None. ''' assert packages_field in ('datasets', 'dataset_count', None) if packages_field == 'dataset_count': dataset_counts = context.get('dataset_counts', None) result_dict = d.table_dictize(group, context) result_dict.update(kw) result_dict['display_name'] = group.title or group.name if include_extras: result_dict['extras'] = extras_dict_dictize(group._extras, context) context['with_capacity'] = True if packages_field: def get_packages_for_this_group(group_, just_the_count=False): # Ask SOLR for the list of packages for this org/group q = { 'facet': 'false', 'rows': 0, } if group_.is_organization: q['fq'] = '+owner_org:"{0}"'.format(group_.id) else: q['fq'] = '+groups:"{0}"'.format(group_.name) # Allow members of organizations to see private datasets. if group_.is_organization: is_group_member = (context.get('user') and authz.has_user_permission_for_group_or_org( group_.id, context.get('user'), 'read')) if is_group_member: q['include_private'] = True if not just_the_count: # package_search limits 'rows' anyway, so this is only if you # want even fewer try: packages_limit = context['limits']['packages'] except KeyError: del q['rows'] # leave it to package_search to limit it else: q['rows'] = packages_limit search_context = dict( (k, v) for (k, v) in context.items() if k != 'schema') search_results = logic.get_action('package_search')(search_context, q) return search_results['count'], search_results['results'] if packages_field == 'datasets': package_count, packages = get_packages_for_this_group(group) result_dict['packages'] = packages else: if dataset_counts is None: package_count, packages = get_packages_for_this_group( group, just_the_count=True) else: # Use the pre-calculated package_counts passed in. facets = dataset_counts if group.is_organization: package_count = facets['owner_org'].get(group.id, 0) else: package_count = facets['groups'].get(group.name, 0) result_dict['package_count'] = package_count if include_tags: # group tags are not creatable via the API yet, but that was(/is) a # future intention (see kindly's commit 5c8df894 on 2011/12/23) result_dict['tags'] = tag_list_dictize( _get_members(context, group, 'tags'), context) if include_groups: # these sub-groups won't have tags or extras for speed result_dict['groups'] = group_list_dictize(_get_members( context, group, 'groups'), context, include_groups=True) if include_users: result_dict['users'] = user_list_dictize( _get_members(context, group, 'users'), context) context['with_capacity'] = False if context.get('for_view'): if result_dict['is_organization']: plugin = plugins.IOrganizationController else: plugin = plugins.IGroupController for item in plugins.PluginImplementations(plugin): result_dict = item.before_view(result_dict) image_url = result_dict.get('image_url') result_dict['image_display_url'] = image_url if image_url and not image_url.startswith('http'): #munge here should not have an effect only doing it incase #of potential vulnerability of dodgy api input image_url = munge.munge_filename_legacy(image_url) result_dict['image_display_url'] = h.url_for_static( 'uploads/group/%s' % result_dict.get('image_url'), qualified=True) return result_dict
def test_01_dictize_main_objects_simple(self): context = {"model": model, "session": model.Session} ## package pkg = model.Session.query( model.Package).filter_by(name='annakarenina').first() result = table_dictize(pkg, context) self.remove_changable_columns(result) assert result == { 'author': None, 'author_email': None, 'license_id': u'other-open', 'maintainer': None, 'maintainer_email': None, 'name': u'annakarenina', 'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n', 'state': u'active', 'title': u'A Novel By Tolstoy', 'url': u'http://www.annakarenina.com', 'version': u'0.7a' }, pprint(result) ## resource resource = pkg.resource_groups[0].resources[0] result = resource_dictize(resource, context) self.remove_changable_columns(result) assert result == { u'alt_url': u'alt123', 'cache_last_updated': None, 'cache_url': None, 'description': u'Full text. Needs escaping: " Umlaut: \xfc', 'format': u'plain text', 'hash': u'abc123', 'last_modified': None, 'mimetype': None, 'mimetype_inner': None, 'name': None, 'position': 0, 'resource_type': None, 'size': None, u'size_extra': u'123', 'state': u'active', 'url': u'http://www.annakarenina.com/download/x=1&y=2', 'webstore_last_updated': None, 'webstore_url': None }, pprint(result) ## package extra key, package_extras = pkg._extras.popitem() result = table_dictize(package_extras, context) self.remove_changable_columns(result) assert result == { 'key': u'genre', 'state': u'active', 'value': u'romantic novel' }, pprint(result)
def package_dictize(pkg, context): ''' Given a Package object, returns an equivalent dictionary. ''' model = context['model'] assert not (context.get('revision_id') or context.get('revision_date')), \ 'Revision functionality is moved to migrate_package_activity' execute = _execute # package if not pkg: raise logic.NotFound result_dict = d.table_dictize(pkg, context) # strip whitespace from title if result_dict.get('title'): result_dict['title'] = result_dict['title'].strip() # resources res = model.resource_table q = select([res]).where(res.c.package_id == pkg.id) result = execute(q, res, context) result_dict["resources"] = resource_list_dictize(result, context) result_dict['num_resources'] = len(result_dict.get('resources', [])) # tags tag = model.tag_table pkg_tag = model.package_tag_table q = select([tag, pkg_tag.c.state], from_obj=pkg_tag.join(tag, tag.c.id == pkg_tag.c.tag_id)).where( pkg_tag.c.package_id == pkg.id) result = execute(q, pkg_tag, context) result_dict["tags"] = d.obj_list_dictize(result, context, lambda x: x["name"]) result_dict['num_tags'] = len(result_dict.get('tags', [])) # Add display_names to tags. At first a tag's display_name is just the # same as its name, but the display_name might get changed later (e.g. # translated into another language by the multilingual extension). for tag in result_dict['tags']: assert 'display_name' not in tag tag['display_name'] = tag['name'] # extras - no longer revisioned, so always provide latest extra = model.package_extra_table q = select([extra]).where(extra.c.package_id == pkg.id) result = execute(q, extra, context) result_dict["extras"] = extras_list_dictize(result, context) # groups member = model.member_table group = model.group_table q = select([group, member.c.capacity], from_obj=member.join(group, group.c.id == member.c.group_id) ).where(member.c.table_id == pkg.id)\ .where(member.c.state == 'active') \ .where(group.c.is_organization == False) result = execute(q, member, context) context['with_capacity'] = False # no package counts as cannot fetch from search index at the same # time as indexing to it. # tags, extras and sub-groups are not included for speed result_dict["groups"] = group_list_dictize(result, context, with_package_counts=False) # owning organization group = model.group_table q = select([group] ).where(group.c.id == pkg.owner_org) \ .where(group.c.state == 'active') result = execute(q, group, context) organizations = d.obj_list_dictize(result, context) if organizations: result_dict["organization"] = organizations[0] else: result_dict["organization"] = None # relations rel = model.package_relationship_table q = select([rel]).where(rel.c.subject_package_id == pkg.id) result = execute(q, rel, context) result_dict["relationships_as_subject"] = \ d.obj_list_dictize(result, context) q = select([rel]).where(rel.c.object_package_id == pkg.id) result = execute(q, rel, context) result_dict["relationships_as_object"] = \ d.obj_list_dictize(result, context) # Extra properties from the domain object # isopen result_dict['isopen'] = pkg.isopen if isinstance(pkg.isopen, bool) \ else pkg.isopen() # type # if null assign the default value to make searching easier result_dict['type'] = pkg.type or u'dataset' # license if pkg.license and pkg.license.url: result_dict['license_url'] = pkg.license.url result_dict['license_title'] = pkg.license.title.split('::')[-1] elif pkg.license: result_dict['license_title'] = pkg.license.title else: result_dict['license_title'] = pkg.license_id # creation and modification date result_dict['metadata_modified'] = pkg.metadata_modified.isoformat() result_dict['metadata_created'] = pkg.metadata_created.isoformat() \ if pkg.metadata_created else None return result_dict
def package_dictize(pkg, context): ''' Given a Package object, returns an equivalent dictionary. Normally this is the current revision (most recent moderated version), but you can provide revision_id, revision_date or pending in the context and it will filter to an earlier time or the latest unmoderated object revision. May raise NotFound. TODO: understand what the specific set of circumstances are that cause this. ''' model = context['model'] #package package_rev = model.package_revision_table q = select([package_rev]).where(package_rev.c.id == pkg.id) result = _execute_with_revision(q, package_rev, context).first() if not result: raise logic.NotFound result_dict = d.table_dictize(result, context) #strip whitespace from title if result_dict.get('title'): result_dict['title'] = result_dict['title'].strip() #resources res_rev = model.resource_revision_table resource_group = model.resource_group_table q = select([res_rev], from_obj = res_rev.join(resource_group, resource_group.c.id == res_rev.c.resource_group_id)) q = q.where(resource_group.c.package_id == pkg.id) result = _execute_with_revision(q, res_rev, context) result_dict["resources"] = resource_list_dictize(result, context) result_dict['num_resources'] = len(result_dict.get('resources', [])) #tags tag_rev = model.package_tag_revision_table tag = model.tag_table q = select([tag, tag_rev.c.state, tag_rev.c.revision_timestamp], from_obj=tag_rev.join(tag, tag.c.id == tag_rev.c.tag_id) ).where(tag_rev.c.package_id == pkg.id) result = _execute_with_revision(q, tag_rev, context) result_dict["tags"] = d.obj_list_dictize(result, context, lambda x: x["name"]) result_dict['num_tags'] = len(result_dict.get('tags', [])) # Add display_names to tags. At first a tag's display_name is just the # same as its name, but the display_name might get changed later (e.g. # translated into another language by the multilingual extension). for tag in result_dict['tags']: assert not tag.has_key('display_name') tag['display_name'] = tag['name'] #extras extra_rev = model.extra_revision_table q = select([extra_rev]).where(extra_rev.c.package_id == pkg.id) result = _execute_with_revision(q, extra_rev, context) result_dict["extras"] = extras_list_dictize(result, context) #groups member_rev = model.member_revision_table group = model.group_table q = select([group, member_rev.c.capacity], from_obj=member_rev.join(group, group.c.id == member_rev.c.group_id) ).where(member_rev.c.table_id == pkg.id)\ .where(member_rev.c.state == 'active') \ .where(group.c.is_organization == False) result = _execute_with_revision(q, member_rev, context) context['with_capacity'] = False ## no package counts as cannot fetch from search index at the same ## time as indexing to it. ## tags, extras and sub-groups are not included for speed result_dict["groups"] = group_list_dictize(result, context, with_package_counts=False) #owning organization group_rev = model.group_revision_table q = select([group_rev] ).where(group_rev.c.id == pkg.owner_org) \ .where(group_rev.c.state == 'active') result = _execute_with_revision(q, group_rev, context) organizations = d.obj_list_dictize(result, context) if organizations: result_dict["organization"] = organizations[0] else: result_dict["organization"] = None #relations rel_rev = model.package_relationship_revision_table q = select([rel_rev]).where(rel_rev.c.subject_package_id == pkg.id) result = _execute_with_revision(q, rel_rev, context) result_dict["relationships_as_subject"] = d.obj_list_dictize(result, context) q = select([rel_rev]).where(rel_rev.c.object_package_id == pkg.id) result = _execute_with_revision(q, rel_rev, context) result_dict["relationships_as_object"] = d.obj_list_dictize(result, context) # Extra properties from the domain object # We need an actual Package object for this, not a PackageRevision if isinstance(pkg, model.PackageRevision): pkg = model.Package.get(pkg.id) # isopen result_dict['isopen'] = pkg.isopen if isinstance(pkg.isopen,bool) else pkg.isopen() # type # if null assign the default value to make searching easier result_dict['type']= pkg.type or u'dataset' # license if pkg.license and pkg.license.url: result_dict['license_url']= pkg.license.url result_dict['license_title']= pkg.license.title.split('::')[-1] elif pkg.license: result_dict['license_title']= pkg.license.title else: result_dict['license_title']= pkg.license_id # creation and modification date result_dict['metadata_modified'] = pkg.metadata_modified.isoformat() result_dict['metadata_created'] = pkg.metadata_created.isoformat() \ if pkg.metadata_created else None return result_dict
def user_following_dataset_dictize(follower, context): return d.table_dictize(follower, context)
def related_dictize(rel, context): return d.table_dictize(rel, context)
def package_dictize_with_revisions(pkg, context): ''' Given a Package object, returns an equivalent dictionary. Normally this is the most recent version, but you can provide revision_id or revision_date in the context and it will filter to an earlier time. May raise NotFound if: * the specified revision_id doesn't exist * the specified revision_date was before the package was created ''' model = context['model'] try: model.PackageRevision # CKAN<=2.8 revision_model = model except AttributeError: # CKAN>2.8 revision_model = RevisionTableMappings.instance() is_latest_revision = not (context.get(u'revision_id') or context.get(u'revision_date')) execute = _execute if is_latest_revision else _execute_with_revision # package if is_latest_revision: if isinstance(pkg, revision_model.PackageRevision): pkg = model.Package.get(pkg.id) result = pkg else: package_rev = revision_model.package_revision_table q = select([package_rev]).where(package_rev.c.id == pkg.id) result = execute(q, package_rev, context).first() if not result: raise logic.NotFound result_dict = d.table_dictize(result, context) # strip whitespace from title if result_dict.get(u'title'): result_dict['title'] = result_dict['title'].strip() # resources if is_latest_revision: res = model.resource_table else: res = revision_model.resource_revision_table # metadata_modified was added after the revisioning was removed so # it does not exist on the resource_revision table. mm_col = res._columns.get(u'metadata_modified') if mm_col is not None: res._columns.remove(mm_col) q = select([res]).where(res.c.package_id == pkg.id) result = execute(q, res, context) result_dict["resources"] = resource_list_dictize(result, context) result_dict['num_resources'] = len(result_dict.get(u'resources', [])) # tags tag = model.tag_table if is_latest_revision: pkg_tag = model.package_tag_table else: pkg_tag = revision_model.package_tag_revision_table q = select([tag, pkg_tag.c.state], from_obj=pkg_tag.join(tag, tag.c.id == pkg_tag.c.tag_id)).where( pkg_tag.c.package_id == pkg.id) result = execute(q, pkg_tag, context) result_dict["tags"] = d.obj_list_dictize(result, context, lambda x: x["name"]) result_dict['num_tags'] = len(result_dict.get(u'tags', [])) # Add display_names to tags. At first a tag's display_name is just the # same as its name, but the display_name might get changed later (e.g. # translated into another language by the multilingual extension). for tag in result_dict['tags']: assert u'display_name' not in tag tag['display_name'] = tag['name'] # extras if is_latest_revision: extra = model.package_extra_table else: extra = revision_model.extra_revision_table q = select([extra]).where(extra.c.package_id == pkg.id) result = execute(q, extra, context) result_dict["extras"] = extras_list_dictize(result, context) # groups if is_latest_revision: member = model.member_table else: member = revision_model.member_revision_table group = model.group_table q = select([group, member.c.capacity], from_obj=member.join(group, group.c.id == member.c.group_id) ).where(member.c.table_id == pkg.id)\ .where(member.c.state == u'active') \ .where(group.c.is_organization == False) # noqa result = execute(q, member, context) context['with_capacity'] = False # no package counts as cannot fetch from search index at the same # time as indexing to it. # tags, extras and sub-groups are not included for speed result_dict["groups"] = group_list_dictize(result, context, with_package_counts=False) # owning organization if is_latest_revision: group = model.group_table else: group = revision_model.group_revision_table q = select([group] ).where(group.c.id == result_dict['owner_org']) \ .where(group.c.state == u'active') result = execute(q, group, context) organizations = d.obj_list_dictize(result, context) if organizations: result_dict["organization"] = organizations[0] else: result_dict["organization"] = None # relations if is_latest_revision: rel = model.package_relationship_table else: rel = revision_model \ .package_relationship_revision_table q = select([rel]).where(rel.c.subject_package_id == pkg.id) result = execute(q, rel, context) result_dict["relationships_as_subject"] = \ d.obj_list_dictize(result, context) q = select([rel]).where(rel.c.object_package_id == pkg.id) result = execute(q, rel, context) result_dict["relationships_as_object"] = \ d.obj_list_dictize(result, context) # Extra properties from the domain object # We need an actual Package object for this, not a PackageRevision # if isinstance(pkg, model.PackageRevision): # pkg = model.Package.get(pkg.id) # isopen result_dict['isopen'] = pkg.isopen if isinstance(pkg.isopen, bool) \ else pkg.isopen() # type # if null assign the default value to make searching easier result_dict['type'] = pkg.type or u'dataset' # license if pkg.license and pkg.license.url: result_dict['license_url'] = pkg.license.url result_dict['license_title'] = pkg.license.title.split(u'::')[-1] elif pkg.license: result_dict['license_title'] = pkg.license.title else: result_dict['license_title'] = pkg.license_id # creation and modification date if is_latest_revision: result_dict['metadata_modified'] = pkg.metadata_modified.isoformat() # (If not is_latest_revision, don't use pkg which is the latest version. # Instead, use the dates already in result_dict that came from the dictized # PackageRevision) result_dict['metadata_created'] = pkg.metadata_created.isoformat() return result_dict
def _group_or_org_delete(context, data_dict, is_org=False): '''Delete a group. You must be authorized to delete the group. :param id: the name or id of the group :type id: string ''' from sqlalchemy import or_ model = context['model'] user = context['user'] id = _get_or_bust(data_dict, 'id') group = model.Group.get(id) context['group'] = group if group is None: raise NotFound('Group was not found.') if is_org: _check_access('organization_delete', context, data_dict) else: _check_access('group_delete', context, data_dict) # organization delete will not occur while all datasets for that org are # not deleted if is_org: datasets = model.Session.query(model.Package) \ .filter_by(owner_org=group.id) \ .filter(model.Package.state != 'deleted') \ .count() if datasets: if not authz.check_config_permission( 'ckan.auth.create_unowned_dataset'): raise ValidationError( _('Organization cannot be deleted while it ' 'still has datasets')) pkg_table = model.package_table # using Core SQLA instead of the ORM should be faster model.Session.execute(pkg_table.update().where( sqla.and_( pkg_table.c.owner_org == group.id, pkg_table.c.state != 'deleted')).values(owner_org=None)) # The group's Member objects are deleted # (including hierarchy connections to parent and children groups) for member in model.Session.query(model.Member).\ filter(or_(model.Member.table_id == id, model.Member.group_id == id)).\ filter(model.Member.state == 'active').all(): member.delete() group.delete() if is_org: activity_type = 'deleted organization' else: activity_type = 'deleted group' activity_dict = { 'user_id': model.User.by_name(six.ensure_text(user)).id, 'object_id': group.id, 'activity_type': activity_type, 'data': { 'group': dictization.table_dictize(group, context) } } activity_create_context = { 'model': model, 'defer_commit': True, 'ignore_auth': True, 'session': context['session'] } _get_action('activity_create')(activity_create_context, activity_dict) if is_org: plugin_type = plugins.IOrganizationController else: plugin_type = plugins.IGroupController for item in plugins.PluginImplementations(plugin_type): item.delete(group) model.repo.commit()
def jsonpatch_dictize(jsonpatch, context): return d.table_dictize(jsonpatch, context)
def task_status_dictize(task_status, context): return d.table_dictize(task_status, context)
def hdx_group_or_org_update(context, data_dict, is_org=False): # Overriding default so that orgs can have multiple images model = context['model'] user = context['user'] session = context['session'] id = _get_or_bust(data_dict, 'id') group = model.Group.get(id) context["group"] = group if group is None: raise NotFound('Group was not found.') if is_org: check_access('organization_update', context, data_dict) else: check_access('group_update', context, data_dict) # get the schema group_plugin = lib_plugins.lookup_group_plugin(group.type) try: schema = group_plugin.form_to_db_schema_options({'type': 'update', 'api': 'api_version' in context, 'context': context}) except AttributeError: schema = group_plugin.form_to_db_schema() try: customization = json.loads(group.extras['customization']) except: customization = {'image_sq': '', 'image_rect': ''} try: data_dict['customization'] = json.loads(data_dict['customization']) except: data_dict['customization'] = {} # If we're removing the image if 'clear_image_sq' in data_dict and data_dict['clear_image_sq']: remove_image(customization['image_sq']) data_dict['customization']['image_sq'] = '' customization['image_rect'] = '' if 'clear_image_rect' in data_dict and data_dict['clear_image_rect']: remove_image(customization['image_rect']) data_dict['customization']['image_rect'] = '' customization['image_rect'] = '' if 'image_sq_upload' in data_dict and data_dict['image_sq_upload'] != '' and data_dict['image_sq_upload'] != None: # Although weird, the FieldStorage instance has a boolean value of False so we need to compare to None # If old image was uploaded remove it if customization['image_sq']: remove_image(customization['image_sq']) upload1 = uploader.Upload('group', customization['image_sq']) upload1.update_data_dict(data_dict, 'image_sq', 'image_sq_upload', 'clear_upload') if 'image_rect_upload' in data_dict and data_dict['image_rect_upload'] != '' \ and data_dict['image_rect_upload'] != None: # Although weird, the FieldStorage instance has a boolean value of False so we need to compare to None if customization['image_rect']: remove_image(customization['image_rect']) upload2 = uploader.Upload('group', customization['image_rect']) upload2.update_data_dict(data_dict, 'image_rect', 'image_rect_upload', 'clear_upload') storage_path = uploader.get_storage_path() ##Rearrange things the way we need them try: if data_dict['image_sq'] != '' and data_dict['image_sq'] != None: data_dict['customization']['image_sq'] = data_dict['image_sq'] else: data_dict['customization']['image_sq'] = customization['image_sq'] except KeyError: data_dict['customization']['image_sq'] = '' try: if data_dict['image_rect'] != '' and data_dict['image_rect'] != None: data_dict['customization']['image_rect'] = data_dict['image_rect'] else: data_dict['customization']['image_rect'] = customization['image_rect'] except KeyError: data_dict['customization']['image_rect'] = '' data_dict['customization'] = json.dumps(data_dict['customization']) if 'api_version' not in context: # old plugins do not support passing the schema so we need # to ensure they still work try: group_plugin.check_data_dict(data_dict, schema) except TypeError: group_plugin.check_data_dict(data_dict) data, errors = _validate(data_dict, schema, context) log.debug('group_update validate_errs=%r user=%s group=%s data_dict=%r', errors, context.get('user'), context.get('group').name if context.get('group') else '', data_dict) if errors: session.rollback() raise ValidationError(errors) rev = model.repo.new_revision() rev.author = user if 'message' in context: rev.message = context['message'] else: rev.message = _(u'REST API: Update object %s') % data.get("name") # when editing an org we do not want to update the packages if using the # new templates. if ((not is_org) and not converters.asbool( config.get('ckan.legacy_templates', False)) and 'api_version' not in context): context['prevent_packages_update'] = True group = model_save.group_dict_save(data, context) if is_org: plugin_type = plugins.IOrganizationController else: plugin_type = plugins.IGroupController for item in plugins.PluginImplementations(plugin_type): item.edit(group) if is_org: activity_type = 'changed organization' else: activity_type = 'changed group' activity_dict = { 'user_id': model.User.by_name(user.decode('utf8')).id, 'object_id': group.id, 'activity_type': activity_type, } # Handle 'deleted' groups. # When the user marks a group as deleted this comes through here as # a 'changed' group activity. We detect this and change it to a 'deleted' # activity. if group.state == u'deleted': if session.query(ckan.model.Activity).filter_by( object_id=group.id, activity_type='deleted').all(): # A 'deleted group' activity for this group has already been # emitted. # FIXME: What if the group was deleted and then activated again? activity_dict = None else: # We will emit a 'deleted group' activity. activity_dict['activity_type'] = 'deleted group' if activity_dict is not None: activity_dict['data'] = { 'group': dictization.table_dictize(group, context) } activity_create_context = { 'model': model, 'user': user, 'defer_commit': True, 'ignore_auth': True, 'session': session } get_action('activity_create')(activity_create_context, activity_dict) # TODO: Also create an activity detail recording what exactly changed # in the group. try: upload1.upload(uploader.get_max_image_size()) except: pass try: upload2.upload(uploader.get_max_image_size()) except: pass if not context.get('defer_commit'): model.repo.commit() return model_dictize.group_dictize(group, context)
def group_dictize(group, context): result_dict = d.table_dictize(group, context) result_dict['display_name'] = group.display_name result_dict['extras'] = extras_dict_dictize(group._extras, context) include_datasets = context.get('include_datasets', True) q = { 'facet': 'false', 'rows': 0, } if group.is_organization: q['fq'] = 'owner_org:"{0}"'.format(group.id) else: q['fq'] = 'groups:"{0}"'.format(group.name) is_group_member = (context.get('user') and new_authz.has_user_permission_for_group_or_org( group.id, context.get('user'), 'read')) if is_group_member: context['ignore_capacity_check'] = True if include_datasets: q['rows'] = 1000 # Only the first 1000 datasets are returned context_ = dict((k, v) for (k, v) in context.items() if k != 'schema') search_results = logic.get_action('package_search')(context_, q) if include_datasets: result_dict['packages'] = search_results['results'] result_dict['package_count'] = search_results['count'] context['with_capacity'] = True result_dict['tags'] = tag_list_dictize( _get_members(context, group, 'tags'), context) result_dict['groups'] = group_list_dictize( _get_members(context, group, 'groups'), context) result_dict['users'] = user_list_dictize( _get_members(context, group, 'users'), context) context['with_capacity'] = False if context.get('for_view'): if result_dict['is_organization']: plugin = plugins.IOrganizationController else: plugin = plugins.IGroupController for item in plugins.PluginImplementations(plugin): result_dict = item.before_view(result_dict) image_url = result_dict.get('image_url') result_dict['image_display_url'] = image_url if image_url and not image_url.startswith('http'): #munge here should not have an effect only doing it incase #of potential vulnerability of dodgy api input image_url = munge.munge_filename(image_url) result_dict['image_display_url'] = h.url_for_static( 'uploads/group/%s' % result_dict.get('image_url'), qualified=True) return result_dict
def package_dictize(pkg, context): ''' Given a Package object, returns an equivalent dictionary. Normally this is the current revision (most recent moderated version), but you can provide revision_id, revision_date or pending in the context and it will filter to an earlier time or the latest unmoderated object revision. May raise NotFound. TODO: understand what the specific set of circumstances are that cause this. ''' model = context['model'] #package package_rev = model.package_revision_table q = select([package_rev]).where(package_rev.c.id == pkg.id) result = _execute_with_revision(q, package_rev, context).first() if not result: raise NotFound result_dict = table_dictize(result, context) #resources res_rev = model.resource_revision_table resource_group = model.resource_group_table q = select([res_rev], from_obj = res_rev.join(resource_group, resource_group.c.id == res_rev.c.resource_group_id)) q = q.where(resource_group.c.package_id == pkg.id) result = _execute_with_revision(q, res_rev, context) result_dict["resources"] = resource_list_dictize(result, context) #tags tag_rev = model.package_tag_revision_table tag = model.tag_table q = select([tag, tag_rev.c.state, tag_rev.c.revision_timestamp], from_obj=tag_rev.join(tag, tag.c.id == tag_rev.c.tag_id) ).where(tag_rev.c.package_id == pkg.id) result = _execute_with_revision(q, tag_rev, context) result_dict["tags"] = obj_list_dictize(result, context, lambda x: x["name"]) #extras extra_rev = model.extra_revision_table q = select([extra_rev]).where(extra_rev.c.package_id == pkg.id) result = _execute_with_revision(q, extra_rev, context) result_dict["extras"] = extras_list_dictize(result, context) #groups member_rev = model.member_revision_table group = model.group_table q = select([group], from_obj=member_rev.join(group, group.c.id == member_rev.c.group_id) ).where(member_rev.c.table_id == pkg.id) result = _execute_with_revision(q, member_rev, context) result_dict["groups"] = obj_list_dictize(result, context) #relations rel_rev = model.package_relationship_revision_table q = select([rel_rev]).where(rel_rev.c.subject_package_id == pkg.id) result = _execute_with_revision(q, rel_rev, context) result_dict["relationships_as_subject"] = obj_list_dictize(result, context) q = select([rel_rev]).where(rel_rev.c.object_package_id == pkg.id) result = _execute_with_revision(q, rel_rev, context) result_dict["relationships_as_object"] = obj_list_dictize(result, context) # Extra properties from the domain object # We need an actual Package object for this, not a PackageRevision if isinstance(pkg,PackageRevision): pkg = model.Package.get(pkg.id) # isopen result_dict['isopen'] = pkg.isopen if isinstance(pkg.isopen,bool) else pkg.isopen() # creation and modification date result_dict['metadata_modified'] = pkg.metadata_modified.isoformat() \ if pkg.metadata_modified else None result_dict['metadata_created'] = pkg.metadata_created.isoformat() \ if pkg.metadata_created else None return result_dict
def package_dictize(pkg, context): ''' Given a Package object, returns an equivalent dictionary. Normally this is the current revision (most recent moderated version), but you can provide revision_id, revision_date or pending in the context and it will filter to an earlier time or the latest unmoderated object revision. May raise NotFound. TODO: understand what the specific set of circumstances are that cause this. ''' model = context['model'] #package package_rev = model.package_revision_table q = select([package_rev]).where(package_rev.c.id == pkg.id) result = _execute_with_revision(q, package_rev, context).first() if not result: raise logic.NotFound result_dict = d.table_dictize(result, context) #resources res_rev = model.resource_revision_table resource_group = model.resource_group_table q = select([res_rev], from_obj = res_rev.join(resource_group, resource_group.c.id == res_rev.c.resource_group_id)) q = q.where(resource_group.c.package_id == pkg.id) result = _execute_with_revision(q, res_rev, context) result_dict["resources"] = resource_list_dictize(result, context) #tags tag_rev = model.package_tag_revision_table tag = model.tag_table q = select([tag, tag_rev.c.state, tag_rev.c.revision_timestamp], from_obj=tag_rev.join(tag, tag.c.id == tag_rev.c.tag_id) ).where(tag_rev.c.package_id == pkg.id) result = _execute_with_revision(q, tag_rev, context) result_dict["tags"] = d.obj_list_dictize(result, context, lambda x: x["name"]) # Add display_names to tags. At first a tag's display_name is just the # same as its name, but the display_name might get changed later (e.g. # translated into another language by the multilingual extension). for tag in result_dict['tags']: assert not tag.has_key('display_name') tag['display_name'] = tag['name'] #extras extra_rev = model.extra_revision_table q = select([extra_rev]).where(extra_rev.c.package_id == pkg.id) result = _execute_with_revision(q, extra_rev, context) result_dict["extras"] = extras_list_dictize(result, context) #tracking tracking = model.TrackingSummary.get_for_package(pkg.id) result_dict['tracking_summary'] = tracking #groups member_rev = model.member_revision_table group = model.group_table q = select([group, member_rev.c.capacity], from_obj=member_rev.join(group, group.c.id == member_rev.c.group_id) ).where(member_rev.c.table_id == pkg.id)\ .where(member_rev.c.state == 'active') result = _execute_with_revision(q, member_rev, context) result_dict["groups"] = d.obj_list_dictize(result, context) #relations rel_rev = model.package_relationship_revision_table q = select([rel_rev]).where(rel_rev.c.subject_package_id == pkg.id) result = _execute_with_revision(q, rel_rev, context) result_dict["relationships_as_subject"] = d.obj_list_dictize(result, context) q = select([rel_rev]).where(rel_rev.c.object_package_id == pkg.id) result = _execute_with_revision(q, rel_rev, context) result_dict["relationships_as_object"] = d.obj_list_dictize(result, context) # Extra properties from the domain object # We need an actual Package object for this, not a PackageRevision if isinstance(pkg, ckan.model.PackageRevision): pkg = model.Package.get(pkg.id) # isopen result_dict['isopen'] = pkg.isopen if isinstance(pkg.isopen,bool) else pkg.isopen() # type result_dict['type']= pkg.type # licence if pkg.license and pkg.license.url: result_dict['license_url']= pkg.license.url result_dict['license_title']= pkg.license.title.split('::')[-1] elif pkg.license: result_dict['license_title']= pkg.license.title else: result_dict['license_title']= pkg.license_id # creation and modification date result_dict['metadata_modified'] = context.pop('metadata_modified') result_dict['metadata_created'] = pkg.metadata_created.isoformat() \ if pkg.metadata_created else None if context.get('for_view'): for item in plugins.PluginImplementations( plugins.IPackageController): result_dict = item.before_view(result_dict) return result_dict
def activity_dictize(activity, context): activity_dict = d.table_dictize(activity, context) return activity_dict
def group_list_dictize(obj_list, context, sort_key=lambda x: x['display_name'], reverse=False, with_package_counts=True): active = context.get('active', True) with_private = context.get('include_private_packages', False) if with_package_counts: query = search.PackageSearchQuery() q = { 'q': '+capacity:public' if not with_private else '*:*', 'fl': 'groups', 'facet.field': ['groups', 'owner_org'], 'facet.limit': -1, 'rows': 1 } query.run(q) result_list = [] for obj in obj_list: if context.get('with_capacity'): obj, capacity = obj group_dict = d.table_dictize(obj, context, capacity=capacity) else: group_dict = d.table_dictize(obj, context) group_dict.pop('created') if active and obj.state not in ('active', 'pending'): continue group_dict['display_name'] = (group_dict.get('title') or group_dict.get('name')) image_url = group_dict.get('image_url') group_dict['image_display_url'] = image_url if image_url and not image_url.startswith('http'): #munge here should not have an effect only doing it incase #of potential vulnerability of dodgy api input image_url = munge.munge_filename(image_url) group_dict['image_display_url'] = h.url_for_static( 'uploads/group/%s' % group_dict.get('image_url'), qualified=True) if with_package_counts: facets = query.facets if obj.is_organization: group_dict['packages'] = facets['owner_org'].get(obj.id, 0) else: group_dict['packages'] = facets['groups'].get(obj.name, 0) if context.get('for_view'): if group_dict['is_organization']: plugin = plugins.IOrganizationController else: plugin = plugins.IGroupController for item in plugins.PluginImplementations(plugin): group_dict = item.before_view(group_dict) result_list.append(group_dict) return sorted(result_list, key=sort_key, reverse=reverse)
def activity_detail_dictize(activity_detail, context): return d.table_dictize(activity_detail, context)
def as_dict(self): context = {'model': model} oapic_dict = dictization.table_dictize(self, context) return oapic_dict