def user_dictize(user, context):
if context.get('with_capacity'):
user, capacity = user
result_dict = d.table_dictize(user, context, capacity=capacity)
else:
result_dict = d.table_dictize(user, context)
del result_dict['password']
result_dict['display_name'] = user.display_name
result_dict['email_hash'] = user.email_hash
result_dict['number_of_edits'] = user.number_of_edits()
result_dict['number_administered_packages'] = user.number_administered_packages()
model = context['model']
session = model.Session
if context.get('with_related'):
related_items = session.query(model.Related).\
filter(model.Related.owner_id==user.id).all()
result_dict['related_items'] = related_list_dictize(related_items,
context)
return result_dict
def group_list_dictize(obj_list, context,
sort_key=lambda x:x['display_name'], reverse=False):
active = context.get('active', True)
with_private = context.get('include_private_packages', False)
result_list = []
for obj in obj_list:
if context.get('with_capacity'):
obj, capacity = obj
group_dict = d.table_dictize(obj, context, capacity=capacity)
else:
group_dict = d.table_dictize(obj, context)
group_dict.pop('created')
if active and obj.state not in ('active', 'pending'):
continue
group_dict['display_name'] = obj.display_name
group_dict['packages'] = \
len(obj.active_packages(with_private=with_private).all())
if context.get('for_view'):
for item in plugins.PluginImplementations(
plugins.IGroupController):
group_dict = item.before_view(group_dict)
result_list.append(group_dict)
return sorted(result_list, key=sort_key, reverse=reverse)
def user_dictize(user, context):
if context.get('with_capacity'):
user, capacity = user
result_dict = d.table_dictize(user, context, capacity=capacity)
else:
result_dict = d.table_dictize(user, context)
del result_dict['password']
result_dict['display_name'] = user.display_name
result_dict['email_hash'] = user.email_hash
result_dict['number_of_edits'] = user.number_of_edits()
result_dict['number_administered_packages'] = user.number_administered_packages()
requester = context.get('user')
if not (new_authz.is_sysadmin(requester) or
requester == user.name or
context.get('keep_sensitive_data', False)):
# If not sysadmin or the same user, strip sensible info
result_dict.pop('apikey', None)
result_dict.pop('reset_key', None)
result_dict.pop('email', None)
model = context['model']
session = model.Session
if context.get('with_related'):
related_items = session.query(model.Related).\
filter(model.Related.owner_id==user.id).all()
result_dict['related_items'] = related_list_dictize(related_items,
context)
return result_dict
def tag_list_dictize(tag_list, context):
result_list = []
for tag in tag_list:
if context.get('with_capacity'):
tag, capacity = tag
dictized = d.table_dictize(tag, context, capacity=capacity)
else:
dictized = d.table_dictize(tag, context)
# Add display_names to tag dicts. At first a tag's display_name is just
# the same as its name, but the display_name might get changed later
# (e.g. translated into another language by the multilingual
# extension).
assert not dictized.has_key('display_name')
dictized['display_name'] = dictized['name']
if context.get('for_view'):
for item in plugins.PluginImplementations(
plugins.ITagController):
dictized = item.before_view(dictized)
result_list.append(dictized)
return result_list
def test_01_dictize_main_objects_simple(self):
context = {"model": model, "session": model.Session}
## package
pkg = model.Session.query(model.Package).filter_by(name="annakarenina").first()
result = table_dictize(pkg, context)
self.remove_changable_columns(result)
assert result == {
"author": None,
"author_email": None,
"license_id": u"other-open",
"maintainer": None,
"maintainer_email": None,
"name": u"annakarenina",
"notes": u"Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n",
"state": u"active",
"title": u"A Novel By Tolstoy",
"url": u"http://www.annakarenina.com",
"version": u"0.7a",
}, pprint(result)
## resource
resource = pkg.resource_groups[0].resources[0]
result = resource_dictize(resource, context)
self.remove_changable_columns(result)
assert result == {
u"alt_url": u"alt123",
"cache_last_updated": None,
"cache_url": None,
"description": u'Full text. Needs escaping: " Umlaut: \xfc',
"format": u"plain text",
"hash": u"abc123",
"last_modified": None,
"mimetype": None,
"mimetype_inner": None,
"name": None,
"position": 0,
"resource_type": None,
"size": None,
u"size_extra": u"123",
"state": u"active",
"url": u"http://www.annakarenina.com/download/x=1&y=2",
"webstore_last_updated": None,
"webstore_url": None,
}, pprint(result)
## package extra
key, package_extras = pkg._extras.popitem()
result = table_dictize(package_extras, context)
self.remove_changable_columns(result)
assert result == {"key": u"genre", "state": u"active", "value": u"romantic novel"}, pprint(result)
def group_list_dictize(obj_list, context,
sort_key=lambda x:x['display_name'], reverse=False,
with_package_counts=True):
active = context.get('active', True)
with_private = context.get('include_private_packages', False)
if with_package_counts:
query = search.PackageSearchQuery()
q = {'q': '+capacity:public' if not with_private else '*:*',
'fl': 'groups', 'facet.field': ['groups', 'owner_org'],
'facet.limit': -1, 'rows': 1}
query.run(q)
result_list = []
for obj in obj_list:
if context.get('with_capacity'):
obj, capacity = obj
group_dict = d.table_dictize(obj, context, capacity=capacity)
else:
group_dict = d.table_dictize(obj, context)
group_dict.pop('created')
if active and obj.state not in ('active', 'pending'):
continue
group_dict['display_name'] = (group_dict.get('title') or
group_dict.get('name'))
image_url = group_dict.get('image_url')
group_dict['image_display_url'] = image_url
if image_url and not image_url.startswith('http'):
#munge here should not have an effect only doing it incase
#of potential vulnerability of dodgy api input
image_url = munge.munge_filename(image_url)
group_dict['image_display_url'] = h.url_for_static(
'uploads/group/%s' % group_dict.get('image_url'),
qualified=True
)
if with_package_counts:
facets = query.facets
if obj.is_organization:
group_dict['packages'] = facets['owner_org'].get(obj.id, 0)
else:
group_dict['packages'] = facets['groups'].get(obj.name, 0)
if context.get('for_view'):
if group_dict['is_organization']:
plugin = plugins.IOrganizationController
else:
plugin = plugins.IGroupController
for item in plugins.PluginImplementations(plugin):
group_dict = item.before_view(group_dict)
result_list.append(group_dict)
return sorted(result_list, key=sort_key, reverse=reverse)
def test_01_dictize_main_objects_simple(self):
context = {"model": model,
"session": model.Session}
## package
pkg = model.Session.query(model.Package).filter_by(name='annakarenina').first()
result = table_dictize(pkg, context)
self.remove_changable_columns(result)
assert result == {
'author': None,
'author_email': None,
'license_id': u'other-open',
'maintainer': None,
'maintainer_email': None,
'name': u'annakarenina',
'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n',
'state': u'active',
'title': u'A Novel By Tolstoy',
'url': u'http://www.annakarenina.com',
'version': u'0.7a'
}, pprint(result)
## resource
resource = pkg.resource_groups[0].resources[0]
result = resource_dictize(resource, context)
self.remove_changable_columns(result)
assert result == {
'alt_url': u'alt123',
'description': u'Full text. Needs escaping: " Umlaut: \xfc',
'format': u'plain text',
'hash': u'abc123',
'position': 0,
'size': u'123',
'state': u'active',
'url': u'http://www.annakarenina.com/download/x=1&y=2'}, pprint(result)
## package extra
key, package_extras = pkg._extras.popitem()
result = table_dictize(package_extras, context)
self.remove_changable_columns(result)
assert result == {
'key': u'genre',
'state': u'active',
'value': u'romantic novel'
}, pprint(result)
def tag_list_dictize(tag_list, context):
result_list = []
for tag in tag_list:
if context.get('with_capacity'):
tag, capacity = tag
dictized = d.table_dictize(tag, context, capacity=capacity)
else:
dictized = d.table_dictize(tag, context)
result_list.append(dictized)
return result_list
def user_dictize(user, context, include_password_hash=False):
if context.get('with_capacity'):
user, capacity = user
result_dict = d.table_dictize(user, context, capacity=capacity)
else:
result_dict = d.table_dictize(user, context)
password_hash = result_dict.pop('password')
del result_dict['reset_key']
result_dict['display_name'] = user.display_name
result_dict['email_hash'] = user.email_hash
result_dict['number_of_edits'] = user.number_of_edits()
result_dict['number_created_packages'] = user.number_created_packages(
include_private_and_draft=context.get(
'count_private_and_draft_datasets', False))
requester = context.get('user')
reset_key = result_dict.pop('reset_key', None)
apikey = result_dict.pop('apikey', None)
email = result_dict.pop('email', None)
if context.get('keep_email', False):
result_dict['email'] = email
if context.get('keep_apikey', False):
result_dict['apikey'] = apikey
if requester == user.name:
result_dict['apikey'] = apikey
result_dict['email'] = email
if authz.is_sysadmin(requester):
result_dict['apikey'] = apikey
result_dict['email'] = email
if include_password_hash:
result_dict['password_hash'] = password_hash
model = context['model']
session = model.Session
if context.get('with_related'):
related_items = session.query(model.Related).\
filter(model.Related.owner_id==user.id).all()
result_dict['related_items'] = related_list_dictize(related_items,
context)
return result_dict
def revision_to_activity(r):
pr = packages_previous.get(r.revision_id)
if only_privatized and (pr is None or (pr.private or not r.private)):
return None
privacy_changed = pr is not None and pr.private != r.private
activity_type = None
if r.state in ('active', 'draft'):
if packages_created[r.id] == r.revision_timestamp:
activity_type = 'new package'
elif privacy_changed:
activity_type = 'changed package privacy'
else:
activity_type = 'changed package'
elif r.state in ('deleted'):
activity_type = 'deleted package'
else:
log.warning("Unknown package state, skipping: %s" % r.state)
return None
d = {'package': dictization.table_dictize(
packages[r.id], context={'model': model})}
activity = model.Activity(r.creator_user_id, r.id, r.revision_id, activity_type, d)
activity.timestamp = r.revision_timestamp
return activity
def vocabulary_dictize(vocabulary, context, include_datasets=False):
vocabulary_dict = d.table_dictize(vocabulary, context)
assert not vocabulary_dict.has_key('tags')
vocabulary_dict['tags'] = [tag_dictize(tag, context, include_datasets)
for tag in vocabulary.tags]
return vocabulary_dict
def current_package_list_with_resources(context, data_dict):
model = context["model"]
user = context["user"]
limit = data_dict.get("limit")
q = ckan.authz.Authorizer().authorized_query(user, model.PackageRevision)
q = q.filter(model.PackageRevision.state=='active')
q = q.filter(model.PackageRevision.current==True)
q = q.order_by(model.package_revision_table.c.revision_timestamp.desc())
if limit:
q = q.limit(limit)
pack_rev = q.all()
package_list = []
for package in pack_rev:
result_dict = table_dictize(package, context)
res_rev = model.resource_revision_table
resource_group = model.resource_group_table
q = select([res_rev], from_obj = res_rev.join(resource_group,
resource_group.c.id == res_rev.c.resource_group_id))
q = q.where(resource_group.c.package_id == package.id)
result = q.where(res_rev.c.current == True).execute()
result_dict["resources"] = resource_list_dictize(result, context)
license_id = result_dict['license_id']
if license_id:
isopen = model.Package.get_license_register()[license_id].isopen()
result_dict['isopen'] = isopen
else:
result_dict['isopen'] = False
package_list.append(result_dict)
return package_list
def resource_dictize(res, context):
resource = d.table_dictize(res, context)
resource['format'] = resource.get('format').lower() if resource.get('format') else ''
extras = resource.pop("extras", None)
if extras:
resource.update(extras)
return resource
def group_dictize(group, context):
model = context["model"]
result_dict = table_dictize(group, context)
result_dict["display_name"] = group.display_name
result_dict["extras"] = extras_dict_dictize(group._extras, context)
context["with_capacity"] = True
result_dict["packages"] = obj_list_dictize(_get_members(context, group, "packages"), context)
result_dict["tags"] = tag_list_dictize(_get_members(context, group, "tags"), context)
result_dict["groups"] = group_list_dictize(_get_members(context, group, "groups"), context)
result_dict["users"] = user_list_dictize(_get_members(context, group, "users"), context)
context["with_capacity"] = False
if context.get("for_view"):
for item in PluginImplementations(IGroupController):
result_dict = item.before_view(result_dict)
return result_dict
def inventory_entry_bulk_create(context, data_dict):
model = context['model']
schema = context['schema']
session = context['session']
organization = model.Group.get(context['organization_name'])
inventory_entry_dict = {'group_id': organization.id}
if not data_dict['field-name-input-0']:
raise ValidationError({'error': [_('Please add at least one inventory entry.')]})
for inventory_entry_id in range(10):
inventory_entry_name = data_dict['field-name-input-' + str(inventory_entry_id)]
if not inventory_entry_name:
break
inventory_entry_dict['title'] = inventory_entry_name
inventory_entry_dict['recurring_interval'] = data_dict['field-recurring-input-' + str(inventory_entry_id)]
inventory_entry_dict['is_recurring'] = (inventory_entry_dict['recurring_interval'] != '0')
data, errors = navl_validate(inventory_entry_dict, schema, context)
if errors:
session.rollback()
# TODO @palcu: i18n
raise ValidationError({'error': [_('Verificati intrarea cu numarul {0}.'.format(inventory_entry_id+1))]})
obj = table_dict_save(inventory_entry_dict, InventoryEntry, context)
model.repo.commit()
return table_dictize(obj, context)
def resource_dictize(res, context):
model = context['model']
resource = d.table_dictize(res, context)
resource_group_id = resource['resource_group_id']
extras = resource.pop("extras", None)
if extras:
resource.update(extras)
# some urls do not have the protocol this adds http:// to these
url = resource['url']
## for_edit is only called at the times when the dataset is to be edited
## in the frontend. Without for_edit the whole qualified url is returned.
if resource.get('url_type') == 'upload' and not context.get('for_edit'):
resource_group = model.Session.query(
model.ResourceGroup).get(resource_group_id)
last_part = url.split('/')[-1]
cleaned_name = munge.munge_filename(last_part)
resource['url'] = h.url_for(controller='package',
action='resource_download',
id=resource_group.package_id,
resource_id=res.id,
filename=cleaned_name,
qualified=True)
elif not urlparse.urlsplit(url).scheme and not context.get('for_edit'):
resource['url'] = u'http://' + url.lstrip('/')
return resource
def inventory_entry_list(context, data_dict):
'''Return a list of inventory entries.
:param name: organization name
:type name: string
:rtype: list of dictionaries
'''
# TODO @palcu: define this
# check_access('inventory_manage', context, data_dict)
model = context['model']
name = get_or_bust(data_dict, 'name')
organization = model.Group.get(name)
if not organization:
raise ObjectNotFound('Organization was not found')
entries = [
table_dictize(entry, context) for entry in organization.inventory_entries]
for entry in entries:
entry['next_deadline_timestamp'] = None
if entry['last_added_dataset_timestamp']:
last_added = _datestamp_to_datetime(entry['last_added_dataset_timestamp'])
delta = timedelta(days=entry['recurring_interval'])
entry['next_deadline_timestamp'] = last_added + delta
return entries
def user_dictize(user, context):
if context.get("with_capacity"):
user, capacity = user
result_dict = table_dictize(user, context, capacity=capacity)
else:
result_dict = table_dictize(user, context)
del result_dict["password"]
result_dict["display_name"] = user.display_name
result_dict["email_hash"] = user.email_hash
result_dict["number_of_edits"] = user.number_of_edits()
result_dict["number_administered_packages"] = user.number_administered_packages()
return result_dict
def tag_dictize(tag, context):
result_dict = table_dictize(tag, context)
result_dict["packages"] = obj_list_dictize(tag.packages, context)
return result_dict
def civicdata_featured_upsert(context, data_dict):
data, errors = df.validate(data_dict, schema, context)
if errors:
raise ValidationError(errors)
featured = db.Civicdata_Featured.get(resource_view_id=data['resource_view_id'])
if featured is None:
featured = db.Civicdata_Featured()
featured.resource_view_id = data['resource_view_id']
if data.has_key('canonical'):
featured.canonical = data['canonical']
if data.has_key('organizationpage'):
featured.organizationpage = data['organizationpage']
resource_id = model.ResourceView.get(featured.resource_view_id).resource_id
featured.package_id = model.Resource.get(resource_id).package_id
featured.save()
session = context['session']
session.add(featured)
session.commit()
return table_dictize(featured, context)
def _package_list_with_resources(context, package_revision_list):
package_list = []
model = context["model"]
for package in package_revision_list:
result_dict = table_dictize(package, context)
res_rev = model.resource_revision_table
resource_group = model.resource_group_table
query = select(
[res_rev], from_obj=res_rev.join(resource_group, resource_group.c.id == res_rev.c.resource_group_id)
)
query = query.where(resource_group.c.package_id == package.id)
result = query.where(res_rev.c.current == True).execute()
result_dict["resources"] = resource_list_dictize(result, context)
license_id = result_dict["license_id"]
if license_id:
try:
isopen = model.Package.get_license_register()[license_id].isopen()
result_dict["isopen"] = isopen
except KeyError:
# TODO: create a log message this error?
result_dict["isopen"] = False
else:
result_dict["isopen"] = False
package_list.append(result_dict)
return package_list
def tag_dictize(tag, context):
tag_dict = d.table_dictize(tag, context)
query = search.PackageSearchQuery()
q = {'q': '+tags:"%s" +capacity:public' % tag.name, 'fl': 'data_dict',
'wt': 'json', 'rows': 1000}
package_dicts = [h.json.loads(result['data_dict']) for result in query.run(q)['results']]
# Add display_names to tags. At first a tag's display_name is just the
# same as its name, but the display_name might get changed later (e.g.
# translated into another language by the multilingual extension).
assert not tag_dict.has_key('display_name')
tag_dict['display_name'] = tag_dict['name']
if context.get('for_view'):
for item in plugins.PluginImplementations(plugins.ITagController):
tag_dict = item.before_view(tag_dict)
tag_dict['packages'] = []
for package_dict in package_dicts:
for item in plugins.PluginImplementations(plugins.IPackageController):
package_dict = item.before_view(package_dict)
tag_dict['packages'].append(package_dict)
else:
tag_dict['packages'] = package_dicts
return tag_dict
def group_dictize(group, context):
model = context['model']
result_dict = d.table_dictize(group, context)
result_dict['display_name'] = group.display_name
result_dict['extras'] = extras_dict_dictize(
group._extras, context)
context['with_capacity'] = True
result_dict['packages'] = d.obj_list_dictize(
_get_members(context, group, 'packages'),
context)
result_dict['tags'] = tag_list_dictize(
_get_members(context, group, 'tags'),
context)
result_dict['groups'] = group_list_dictize(
_get_members(context, group, 'groups'),
context)
result_dict['users'] = user_list_dictize(
_get_members(context, group, 'users'),
context)
context['with_capacity'] = False
if context.get('for_view'):
for item in plugins.PluginImplementations(plugins.IGroupController):
result_dict = item.before_view(result_dict)
return result_dict
def user_dictize(user, context):
if context.get('with_capacity'):
user, capacity = user
result_dict = d.table_dictize(user, context, capacity=capacity)
else:
result_dict = d.table_dictize(user, context)
del result_dict['password']
result_dict['display_name'] = user.display_name
result_dict['email_hash'] = user.email_hash
result_dict['number_of_edits'] = user.number_of_edits()
result_dict['number_administered_packages'] = user.number_administered_packages()
return result_dict
def group_dictize(group, context):
model = context['model']
result_dict = table_dictize(group, context)
result_dict['display_name'] = group.display_name
result_dict['extras'] = extras_dict_dictize(
group._extras, context)
context['with_capacity'] = True
result_dict['packages'] = obj_list_dictize(
_get_members(context, group, 'packages'),
context)
result_dict['tags'] = tag_list_dictize(
_get_members(context, group, 'tags'),
context)
result_dict['groups'] = group_list_dictize(
_get_members(context, group, 'groups'),
context)
result_dict['users'] = user_list_dictize(
_get_members(context, group, 'users'),
context)
context['with_capacity'] = False
return result_dict
def activity_stream_item(self, activity_type, revision, user_id):
import ckan.model
import ckan.logic
assert activity_type in ("new", "changed"), (
str(activity_type))
# Handle 'deleted' objects.
# When the user marks a package as deleted this comes through here as
# a 'changed' package activity. We detect this and change it to a
# 'deleted' activity.
if activity_type == 'changed' and self.state == u'deleted':
if meta.Session.query(activity.Activity).filter_by(
object_id=self.id, activity_type='deleted').all():
# A 'deleted' activity for this object has already been emitted
# FIXME: What if the object was deleted and then activated
# again?
return None
else:
# Emit a 'deleted' activity for this object.
activity_type = 'deleted'
try:
d = {'package': dictization.table_dictize(self,
context={'model': ckan.model})}
return activity.Activity(user_id, self.id, revision.id,
"%s package" % activity_type, d)
except ckan.logic.NotFound:
# This happens if this package is being purged and therefore has no
# current revision.
# TODO: Purge all related activity stream items when a model object
# is purged.
return None
def user_dictize(user, context):
if context.get('with_capacity'):
user, capacity = user
result_dict = d.table_dictize(user, context, capacity=capacity)
else:
result_dict = d.table_dictize(user, context)
del result_dict['password']
del result_dict['reset_key']
result_dict['display_name'] = user.display_name
result_dict['email_hash'] = user.email_hash
result_dict['number_of_edits'] = user.number_of_edits()
result_dict['number_administered_packages'] = user.number_administered_packages()
requester = context.get('user')
reset_key = result_dict.pop('reset_key', None)
apikey = result_dict.pop('apikey', None)
email = result_dict.pop('email', None)
if context.get('keep_email', False):
result_dict['email'] = email
if context.get('keep_apikey', False):
result_dict['apikey'] = apikey
if requester == user.name:
result_dict['apikey'] = apikey
result_dict['email'] = email
## this should not really really be needed but tests need it
if new_authz.is_sysadmin(requester):
result_dict['apikey'] = apikey
result_dict['email'] = email
model = context['model']
session = model.Session
if context.get('with_related'):
related_items = session.query(model.Related).\
filter(model.Related.owner_id==user.id).all()
result_dict['related_items'] = related_list_dictize(related_items,
context)
return result_dict
def group_dictize(group, context):
model = context['model']
result_dict = d.table_dictize(group, context)
result_dict['display_name'] = group.display_name
result_dict['extras'] = extras_dict_dictize(
group._extras, context)
context['with_capacity'] = True
result_dict['packages'] = d.obj_list_dictize(
_get_members(context, group, 'packages'),
context)
query = search.PackageSearchQuery()
if group.is_organization:
q = {'q': 'owner_org:"%s" +capacity:public' % group.id, 'rows': 1}
else:
q = {'q': 'groups:"%s" +capacity:public' % group.name, 'rows': 1}
package_count = query.run(q)['count']
result_dict['package_count'] = package_count
# if group.is_organization:
# q = {'q': 'owner_org:"%s" +capacity:public' % group.id, 'rows': package_count}
# packages = query.run(q)['results']
## log.error(str(packages))
# result_dict['packages'] = packages
result_dict['tags'] = tag_list_dictize(
_get_members(context, group, 'tags'),
context)
result_dict['groups'] = group_list_dictize(
_get_members(context, group, 'groups'),
context)
if context.get('for_view'):
context['user'] = '******'
result_dict['users'] = user_list_dictize(
_get_members(context, group, 'users'),
context)
context['with_capacity'] = False
if context.get('for_view'):
if result_dict['is_organization']:
plugin = plugins.IOrganizationController
else:
plugin = plugins.IGroupController
for item in plugins.PluginImplementations(plugin):
result_dict = item.before_view(result_dict)
return result_dict
def test_07_table_simple_save(self):
context = {"model": model,
"session": model.Session}
anna1 = model.Session.query(model.Package).filter_by(name='annakarenina').one()
anna_dictized = self.remove_changable_columns(table_dictize(anna1, context))
anna_dictized["name"] = 'annakarenina2'
model.repo.new_revision()
table_dict_save(anna_dictized, model.Package, context)
model.Session.commit()
pkg = model.Session.query(model.Package).filter_by(name='annakarenina2').one()
assert self.remove_changable_columns(table_dictize(pkg, context)) == anna_dictized, self.remove_changable_columns(table_dictize(pkg, context))
def resource_view_dictize(resource_view, context):
dictized = d.table_dictize(resource_view, context)
dictized.pop('order')
config = dictized.pop('config', {})
dictized.update(config)
resource = context['model'].Resource.get(resource_view.resource_id)
package_id = resource.package_id
dictized['package_id'] = package_id
return dictized
def as_dict(self):
context = {'model': model}
rating_dict = dictization.table_dictize(self, context)
return rating_dict
def _group_or_org_update(context, data_dict, is_org=False):
model = context['model']
user = context['user']
session = context['session']
id = _get_or_bust(data_dict, 'id')
group = model.Group.get(id)
context["group"] = group
if group is None:
raise NotFound('Group was not found.')
data_dict['type'] = group.type
# get the schema
group_plugin = lib_plugins.lookup_group_plugin(group.type)
try:
schema = group_plugin.form_to_db_schema_options({
'type':
'update',
'api':
'api_version' in context,
'context':
context
})
except AttributeError:
schema = group_plugin.form_to_db_schema()
upload = uploader.get_uploader('group', group.image_url)
upload.update_data_dict(data_dict, 'image_url', 'image_upload',
'clear_upload')
if is_org:
_check_access('organization_update', context, data_dict)
else:
_check_access('group_update', context, data_dict)
if 'api_version' not in context:
# old plugins do not support passing the schema so we need
# to ensure they still work
try:
group_plugin.check_data_dict(data_dict, schema)
except TypeError:
group_plugin.check_data_dict(data_dict)
data, errors = lib_plugins.plugin_validate(
group_plugin, context, data_dict, schema,
'organization_update' if is_org else 'group_update')
log.debug('group_update validate_errs=%r user=%s group=%s data_dict=%r',
errors, context.get('user'),
context.get('group').name if context.get('group') else '',
data_dict)
if errors:
session.rollback()
raise ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name")
group = model_save.group_dict_save(data,
context,
prevent_packages_update=is_org)
if is_org:
plugin_type = plugins.IOrganizationController
else:
plugin_type = plugins.IGroupController
for item in plugins.PluginImplementations(plugin_type):
item.edit(group)
if is_org:
activity_type = 'changed organization'
else:
activity_type = 'changed group'
activity_dict = {
'user_id': model.User.by_name(user.decode('utf8')).id,
'object_id': group.id,
'activity_type': activity_type,
}
# Handle 'deleted' groups.
# When the user marks a group as deleted this comes through here as
# a 'changed' group activity. We detect this and change it to a 'deleted'
# activity.
if group.state == u'deleted':
if session.query(ckan.model.Activity).filter_by(
object_id=group.id, activity_type='deleted').all():
# A 'deleted group' activity for this group has already been
# emitted.
# FIXME: What if the group was deleted and then activated again?
activity_dict = None
else:
# We will emit a 'deleted group' activity.
activity_dict['activity_type'] = 'deleted group'
if activity_dict is not None:
activity_dict['data'] = {
'group': dictization.table_dictize(group, context)
}
activity_create_context = {
'model': model,
'user': user,
'defer_commit': True,
'ignore_auth': True,
'session': session
}
_get_action('activity_create')(activity_create_context, activity_dict)
# TODO: Also create an activity detail recording what exactly changed
# in the group.
upload.upload(uploader.get_max_image_size())
if not context.get('defer_commit'):
model.repo.commit()
return model_dictize.group_dictize(group, context)
def vocabulary_dictize(vocabulary, context):
vocabulary_dict = d.table_dictize(vocabulary, context)
assert not vocabulary_dict.has_key('tags')
vocabulary_dict['tags'] = [tag_dictize(tag, context) for tag
in vocabulary.tags]
return vocabulary_dict
def resource_dictize(res, context):
resource = d.table_dictize(res, context)
extras = resource.pop("extras", None)
if extras:
resource.update(extras)
return resource
def for_json(self, context):
return table_dictize(self, context)
def user_following_group_dictize(follower, context):
return d.table_dictize(follower, context)
def _group_or_org_update(context, data_dict, is_org=False):
model = context['model']
user = context['user']
session = context['session']
id = _get_or_bust(data_dict, 'id')
parent = context.get('parent', None)
group = model.Group.get(id)
context["group"] = group
if group is None:
raise NotFound('Group was not found.')
# get the schema
group_plugin = lib_plugins.lookup_group_plugin(group.type)
try:
schema = group_plugin.form_to_db_schema_options({
'type':
'update',
'api':
'api_version' in context,
'context':
context
})
except AttributeError:
schema = group_plugin.form_to_db_schema()
if is_org:
_check_access('organization_update', context, data_dict)
else:
_check_access('group_update', context, data_dict)
if 'api_version' not in context:
# old plugins do not support passing the schema so we need
# to ensure they still work
try:
group_plugin.check_data_dict(data_dict, schema)
except TypeError:
group_plugin.check_data_dict(data_dict)
data, errors = _validate(data_dict, schema, context)
log.debug('group_update validate_errs=%r user=%s group=%s data_dict=%r',
errors, context.get('user'),
context.get('group').name if context.get('group') else '',
data_dict)
if errors:
session.rollback()
raise ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name")
# when editing an org we do not want to update the packages if using the
# new templates.
if ((not is_org) and
not converters.asbool(config.get('ckan.legacy_templates', False))
and 'api_version' not in context):
context['prevent_packages_update'] = True
group = model_save.group_dict_save(data, context)
if parent:
parent_group = model.Group.get(parent)
if parent_group and not parent_group in group.get_groups(group.type):
# Delete all of this groups memberships
current = session.query(model.Member).\
filter(model.Member.table_id == group.id).\
filter(model.Member.table_name == "group").all()
if current:
log.debug('Parents of group %s deleted: %r', group.name,
[membership.group.name for membership in current])
for c in current:
session.delete(c)
member = model.Member(group=parent_group,
table_id=group.id,
table_name='group')
session.add(member)
log.debug('Group %s is made child of group %s', group.name,
parent_group.name)
if is_org:
plugin_type = plugins.IOrganizationController
else:
plugin_type = plugins.IGroupController
for item in plugins.PluginImplementations(plugin_type):
item.edit(group)
if is_org:
activity_type = 'changed organization'
else:
activity_type = 'changed group'
activity_dict = {
'user_id': model.User.by_name(user.decode('utf8')).id,
'object_id': group.id,
'activity_type': activity_type,
}
# Handle 'deleted' groups.
# When the user marks a group as deleted this comes through here as
# a 'changed' group activity. We detect this and change it to a 'deleted'
# activity.
if group.state == u'deleted':
if session.query(ckan.model.Activity).filter_by(
object_id=group.id, activity_type='deleted').all():
# A 'deleted group' activity for this group has already been
# emitted.
# FIXME: What if the group was deleted and then activated again?
activity_dict = None
else:
# We will emit a 'deleted group' activity.
activity_dict['activity_type'] = 'deleted group'
if activity_dict is not None:
activity_dict['data'] = {
'group': dictization.table_dictize(group, context)
}
activity_create_context = {
'model': model,
'user': user,
'defer_commit': True,
'session': session
}
_get_action('activity_create')(activity_create_context,
activity_dict,
ignore_auth=True)
# TODO: Also create an activity detail recording what exactly changed
# in the group.
if not context.get('defer_commit'):
model.repo.commit()
return model_dictize.group_dictize(group, context)
def tag_dictize(tag, context):
result_dict = d.table_dictize(tag, context)
result_dict["packages"] = d.obj_list_dictize(tag.packages, context)
return result_dict
def group_dictize(group, context,
include_groups=True,
include_tags=True,
include_users=True,
include_extras=True,
packages_field='datasets',
**kw):
'''
Turns a Group object and related into a dictionary. The related objects
like tags are included unless you specify it in the params.
:param packages_field: determines the format of the `packages` field - can
be `datasets`, `dataset_count`, `none_but_include_package_count` or None.
If set to `dataset_count` or `none_but_include_package_count` then you
can precalculate dataset counts in advance by supplying:
context['dataset_counts'] = get_group_dataset_counts()
'''
assert packages_field in ('datasets', 'dataset_count',
'none_but_include_package_count', None)
if packages_field in ('dataset_count', 'none_but_include_package_count'):
dataset_counts = context.get('dataset_counts', None)
result_dict = d.table_dictize(group, context)
result_dict.update(kw)
result_dict['display_name'] = group.title or group.name
if include_extras:
result_dict['extras'] = extras_dict_dictize(
group._extras, context)
context['with_capacity'] = True
if packages_field:
def get_packages_for_this_group(group_):
# Ask SOLR for the list of packages for this org/group
q = {
'facet': 'false',
'rows': 0,
}
if group_.is_organization:
q['fq'] = 'owner_org:"{0}"'.format(group_.id)
else:
q['fq'] = 'groups:"{0}"'.format(group_.name)
is_group_member = (context.get('user') and
new_authz.has_user_permission_for_group_or_org(group_.id, context.get('user'), 'read'))
if is_group_member:
context['ignore_capacity_check'] = True
if not context.get('for_view'):
q['rows'] = 1000 # Only the first 1000 datasets are returned
search_context = dict((k, v) for (k, v) in context.items() if k != 'schema')
search_results = logic.get_action('package_search')(search_context, q)
return search_results['count'], search_results['results']
if packages_field == 'datasets':
package_count, packages = get_packages_for_this_group(group)
result_dict['packages'] = packages
else:
# i.e. packages_field is 'dataset_count' or
# 'none_but_include_package_count'
if dataset_counts is None:
package_count, packages = get_packages_for_this_group(group)
else:
# Use the pre-calculated package_counts passed in.
facets = dataset_counts
if group.is_organization:
package_count = facets['owner_org'].get(group.id, 0)
else:
package_count = facets['groups'].get(group.name, 0)
if packages_field != 'none_but_include_package_count':
result_dict['packages'] = package_count
result_dict['package_count'] = package_count
if include_tags:
# group tags are not creatable via the API yet, but that was(/is) a
# future intention (see kindly's commit 5c8df894 on 2011/12/23)
result_dict['tags'] = tag_list_dictize(
_get_members(context, group, 'tags'),
context)
if include_groups:
# these sub-groups won't have tags or extras for speed
result_dict['groups'] = group_list_dictize(
_get_members(context, group, 'groups'),
context, include_groups=True)
if include_users:
result_dict['users'] = user_list_dictize(
_get_members(context, group, 'users'),
context)
context['with_capacity'] = False
if context.get('for_view'):
if result_dict['is_organization']:
plugin = plugins.IOrganizationController
else:
plugin = plugins.IGroupController
for item in plugins.PluginImplementations(plugin):
result_dict = item.before_view(result_dict)
image_url = result_dict.get('image_url')
result_dict['image_display_url'] = image_url
if image_url and not image_url.startswith('http'):
#munge here should not have an effect only doing it incase
#of potential vulnerability of dodgy api input
image_url = munge.munge_filename(image_url)
result_dict['image_display_url'] = h.url_for_static(
'uploads/group/%s' % result_dict.get('image_url'),
qualified = True
)
return result_dict
def member_dictize(member, context):
return d.table_dictize(member, context)
def notify_maintainer(pkg_dict, maintainer):
if not is_maintainer_notification():
return
object_id_validators[activity_assign_maintainer] = tk.get_validator(
'user_id_exists')
context = {
'model': model,
'session': ckan.model.Session,
'user': tk.c.user or tk.c.author,
'ignore_auth': True
}
admin_user = tk.get_action('get_site_user')(context, {})
dataset_context = {
'model': model,
'session': ckan.model.Session,
'user': tk.c.user or tk.c.author,
'api_version': 3,
'for_edit': True,
'auth_user_obj': admin_user,
'ignore_auth': True
}
activity_data = {}
activity_object_id = pkg_dict.get('id')
if not activity_object_id:
return
#modification check
package = model.Package.get(activity_object_id)
package_revision = model.package.package_revision_table
cols = [
package_revision.c.maintainer, package_revision.c.metadata_modified
]
sql = select(cols, from_obj=[
package_revision
]).where(package_revision.c.id == activity_object_id).order_by(
package_revision.c.revision_timestamp.desc())
rows = model.Session.execute(sql).fetchall()
if rows[0]['metadata_modified'] != package.metadata_modified:
return #no updated content in package table, including maintainer
if len(rows) > 1 and rows[1]['maintainer'] == maintainer.name:
return #if maintainer is the same as before
#retrieve package data as activity_data
try:
activity_data = tk.get_action('package_show')(dataset_context, {
'id': activity_object_id
})
except NotFound:
#at new creation of the package, the data has not been existing in the DB, so retrieve the data from pkg_dict and sanitize it
activity_data = copy.deepcopy(pkg_dict)
_sanitize_dict(activity_data) #sanitize
maintainer_dict = table_dictize(maintainer, context)
#add item into the creators activity stream indicating the maintainer has been assigned
activity_dict = {
'user_id': tk.c.user or tk.c.author,
'object_id': maintainer.id,
'data': {
'dataset': activity_data,
'user': maintainer_dict
},
'activity_type': activity_assign_maintainer,
}
tk.get_action('activity_create')(context, activity_dict)
print 'done'
def user_dictize(user,
context,
include_password_hash=False,
include_plugin_extras=False):
if context.get('with_capacity'):
user, capacity = user
result_dict = d.table_dictize(user, context, capacity=capacity)
else:
result_dict = d.table_dictize(user, context)
password_hash = result_dict.pop('password')
del result_dict['reset_key']
result_dict['display_name'] = user.display_name
result_dict['email_hash'] = user.email_hash
result_dict['number_created_packages'] = user.number_created_packages(
include_private_and_draft=context.get(
'count_private_and_draft_datasets', False))
requester = context.get('user')
reset_key = result_dict.pop('reset_key', None)
apikey = result_dict.pop('apikey', None)
email = result_dict.pop('email', None)
plugin_extras = result_dict.pop('plugin_extras', None)
if context.get('keep_email', False):
result_dict['email'] = email
if context.get('keep_apikey', False):
result_dict['apikey'] = apikey
if requester == user.name:
result_dict['apikey'] = apikey
result_dict['email'] = email
if authz.is_sysadmin(requester):
result_dict['apikey'] = apikey
result_dict['email'] = email
if include_password_hash:
result_dict['password_hash'] = password_hash
if include_plugin_extras:
result_dict['plugin_extras'] = copy.deepcopy(
plugin_extras) if plugin_extras else plugin_extras
model = context['model']
session = model.Session
image_url = result_dict.get('image_url')
result_dict['image_display_url'] = image_url
if image_url and not image_url.startswith('http'):
# munge here should not have any effect, only doing it in case
# of potential vulnerability of dodgy api input.
image_url = munge.munge_filename_legacy(image_url)
result_dict['image_display_url'] = h.url_for_static(
'uploads/user/%s' % result_dict.get('image_url'), qualified=True)
return result_dict
def group_dictize(group,
context,
include_groups=True,
include_tags=True,
include_users=True,
include_extras=True,
packages_field='datasets',
**kw):
'''
Turns a Group object and related into a dictionary. The related objects
like tags are included unless you specify it in the params.
:param packages_field: determines the format of the `packages` field - can
be `datasets`, `dataset_count` or None.
'''
assert packages_field in ('datasets', 'dataset_count', None)
if packages_field == 'dataset_count':
dataset_counts = context.get('dataset_counts', None)
result_dict = d.table_dictize(group, context)
result_dict.update(kw)
result_dict['display_name'] = group.title or group.name
if include_extras:
result_dict['extras'] = extras_dict_dictize(group._extras, context)
context['with_capacity'] = True
if packages_field:
def get_packages_for_this_group(group_, just_the_count=False):
# Ask SOLR for the list of packages for this org/group
q = {
'facet': 'false',
'rows': 0,
}
if group_.is_organization:
q['fq'] = '+owner_org:"{0}"'.format(group_.id)
else:
q['fq'] = '+groups:"{0}"'.format(group_.name)
# Allow members of organizations to see private datasets.
if group_.is_organization:
is_group_member = (context.get('user') and
authz.has_user_permission_for_group_or_org(
group_.id, context.get('user'), 'read'))
if is_group_member:
q['include_private'] = True
if not just_the_count:
# package_search limits 'rows' anyway, so this is only if you
# want even fewer
try:
packages_limit = context['limits']['packages']
except KeyError:
del q['rows'] # leave it to package_search to limit it
else:
q['rows'] = packages_limit
search_context = dict(
(k, v) for (k, v) in context.items() if k != 'schema')
search_results = logic.get_action('package_search')(search_context,
q)
return search_results['count'], search_results['results']
if packages_field == 'datasets':
package_count, packages = get_packages_for_this_group(group)
result_dict['packages'] = packages
else:
if dataset_counts is None:
package_count, packages = get_packages_for_this_group(
group, just_the_count=True)
else:
# Use the pre-calculated package_counts passed in.
facets = dataset_counts
if group.is_organization:
package_count = facets['owner_org'].get(group.id, 0)
else:
package_count = facets['groups'].get(group.name, 0)
result_dict['package_count'] = package_count
if include_tags:
# group tags are not creatable via the API yet, but that was(/is) a
# future intention (see kindly's commit 5c8df894 on 2011/12/23)
result_dict['tags'] = tag_list_dictize(
_get_members(context, group, 'tags'), context)
if include_groups:
# these sub-groups won't have tags or extras for speed
result_dict['groups'] = group_list_dictize(_get_members(
context, group, 'groups'),
context,
include_groups=True)
if include_users:
result_dict['users'] = user_list_dictize(
_get_members(context, group, 'users'), context)
context['with_capacity'] = False
if context.get('for_view'):
if result_dict['is_organization']:
plugin = plugins.IOrganizationController
else:
plugin = plugins.IGroupController
for item in plugins.PluginImplementations(plugin):
result_dict = item.before_view(result_dict)
image_url = result_dict.get('image_url')
result_dict['image_display_url'] = image_url
if image_url and not image_url.startswith('http'):
#munge here should not have an effect only doing it incase
#of potential vulnerability of dodgy api input
image_url = munge.munge_filename_legacy(image_url)
result_dict['image_display_url'] = h.url_for_static(
'uploads/group/%s' % result_dict.get('image_url'), qualified=True)
return result_dict
def test_01_dictize_main_objects_simple(self):
context = {"model": model, "session": model.Session}
## package
pkg = model.Session.query(
model.Package).filter_by(name='annakarenina').first()
result = table_dictize(pkg, context)
self.remove_changable_columns(result)
assert result == {
'author': None,
'author_email': None,
'license_id': u'other-open',
'maintainer': None,
'maintainer_email': None,
'name': u'annakarenina',
'notes':
u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n',
'state': u'active',
'title': u'A Novel By Tolstoy',
'url': u'http://www.annakarenina.com',
'version': u'0.7a'
}, pprint(result)
## resource
resource = pkg.resource_groups[0].resources[0]
result = resource_dictize(resource, context)
self.remove_changable_columns(result)
assert result == {
u'alt_url': u'alt123',
'cache_last_updated': None,
'cache_url': None,
'description': u'Full text. Needs escaping: " Umlaut: \xfc',
'format': u'plain text',
'hash': u'abc123',
'last_modified': None,
'mimetype': None,
'mimetype_inner': None,
'name': None,
'position': 0,
'resource_type': None,
'size': None,
u'size_extra': u'123',
'state': u'active',
'url': u'http://www.annakarenina.com/download/x=1&y=2',
'webstore_last_updated': None,
'webstore_url': None
}, pprint(result)
## package extra
key, package_extras = pkg._extras.popitem()
result = table_dictize(package_extras, context)
self.remove_changable_columns(result)
assert result == {
'key': u'genre',
'state': u'active',
'value': u'romantic novel'
}, pprint(result)
def package_dictize(pkg, context):
'''
Given a Package object, returns an equivalent dictionary.
'''
model = context['model']
assert not (context.get('revision_id') or
context.get('revision_date')), \
'Revision functionality is moved to migrate_package_activity'
execute = _execute
# package
if not pkg:
raise logic.NotFound
result_dict = d.table_dictize(pkg, context)
# strip whitespace from title
if result_dict.get('title'):
result_dict['title'] = result_dict['title'].strip()
# resources
res = model.resource_table
q = select([res]).where(res.c.package_id == pkg.id)
result = execute(q, res, context)
result_dict["resources"] = resource_list_dictize(result, context)
result_dict['num_resources'] = len(result_dict.get('resources', []))
# tags
tag = model.tag_table
pkg_tag = model.package_tag_table
q = select([tag, pkg_tag.c.state],
from_obj=pkg_tag.join(tag, tag.c.id == pkg_tag.c.tag_id)).where(
pkg_tag.c.package_id == pkg.id)
result = execute(q, pkg_tag, context)
result_dict["tags"] = d.obj_list_dictize(result, context,
lambda x: x["name"])
result_dict['num_tags'] = len(result_dict.get('tags', []))
# Add display_names to tags. At first a tag's display_name is just the
# same as its name, but the display_name might get changed later (e.g.
# translated into another language by the multilingual extension).
for tag in result_dict['tags']:
assert 'display_name' not in tag
tag['display_name'] = tag['name']
# extras - no longer revisioned, so always provide latest
extra = model.package_extra_table
q = select([extra]).where(extra.c.package_id == pkg.id)
result = execute(q, extra, context)
result_dict["extras"] = extras_list_dictize(result, context)
# groups
member = model.member_table
group = model.group_table
q = select([group, member.c.capacity],
from_obj=member.join(group, group.c.id == member.c.group_id)
).where(member.c.table_id == pkg.id)\
.where(member.c.state == 'active') \
.where(group.c.is_organization == False)
result = execute(q, member, context)
context['with_capacity'] = False
# no package counts as cannot fetch from search index at the same
# time as indexing to it.
# tags, extras and sub-groups are not included for speed
result_dict["groups"] = group_list_dictize(result,
context,
with_package_counts=False)
# owning organization
group = model.group_table
q = select([group]
).where(group.c.id == pkg.owner_org) \
.where(group.c.state == 'active')
result = execute(q, group, context)
organizations = d.obj_list_dictize(result, context)
if organizations:
result_dict["organization"] = organizations[0]
else:
result_dict["organization"] = None
# relations
rel = model.package_relationship_table
q = select([rel]).where(rel.c.subject_package_id == pkg.id)
result = execute(q, rel, context)
result_dict["relationships_as_subject"] = \
d.obj_list_dictize(result, context)
q = select([rel]).where(rel.c.object_package_id == pkg.id)
result = execute(q, rel, context)
result_dict["relationships_as_object"] = \
d.obj_list_dictize(result, context)
# Extra properties from the domain object
# isopen
result_dict['isopen'] = pkg.isopen if isinstance(pkg.isopen, bool) \
else pkg.isopen()
# type
# if null assign the default value to make searching easier
result_dict['type'] = pkg.type or u'dataset'
# license
if pkg.license and pkg.license.url:
result_dict['license_url'] = pkg.license.url
result_dict['license_title'] = pkg.license.title.split('::')[-1]
elif pkg.license:
result_dict['license_title'] = pkg.license.title
else:
result_dict['license_title'] = pkg.license_id
# creation and modification date
result_dict['metadata_modified'] = pkg.metadata_modified.isoformat()
result_dict['metadata_created'] = pkg.metadata_created.isoformat() \
if pkg.metadata_created else None
return result_dict
def package_dictize(pkg, context):
'''
Given a Package object, returns an equivalent dictionary.
Normally this is the current revision (most recent moderated version),
but you can provide revision_id, revision_date or pending in the
context and it will filter to an earlier time or the latest unmoderated
object revision.
May raise NotFound. TODO: understand what the specific set of
circumstances are that cause this.
'''
model = context['model']
#package
package_rev = model.package_revision_table
q = select([package_rev]).where(package_rev.c.id == pkg.id)
result = _execute_with_revision(q, package_rev, context).first()
if not result:
raise logic.NotFound
result_dict = d.table_dictize(result, context)
#strip whitespace from title
if result_dict.get('title'):
result_dict['title'] = result_dict['title'].strip()
#resources
res_rev = model.resource_revision_table
resource_group = model.resource_group_table
q = select([res_rev], from_obj = res_rev.join(resource_group,
resource_group.c.id == res_rev.c.resource_group_id))
q = q.where(resource_group.c.package_id == pkg.id)
result = _execute_with_revision(q, res_rev, context)
result_dict["resources"] = resource_list_dictize(result, context)
result_dict['num_resources'] = len(result_dict.get('resources', []))
#tags
tag_rev = model.package_tag_revision_table
tag = model.tag_table
q = select([tag, tag_rev.c.state, tag_rev.c.revision_timestamp],
from_obj=tag_rev.join(tag, tag.c.id == tag_rev.c.tag_id)
).where(tag_rev.c.package_id == pkg.id)
result = _execute_with_revision(q, tag_rev, context)
result_dict["tags"] = d.obj_list_dictize(result, context, lambda x: x["name"])
result_dict['num_tags'] = len(result_dict.get('tags', []))
# Add display_names to tags. At first a tag's display_name is just the
# same as its name, but the display_name might get changed later (e.g.
# translated into another language by the multilingual extension).
for tag in result_dict['tags']:
assert not tag.has_key('display_name')
tag['display_name'] = tag['name']
#extras
extra_rev = model.extra_revision_table
q = select([extra_rev]).where(extra_rev.c.package_id == pkg.id)
result = _execute_with_revision(q, extra_rev, context)
result_dict["extras"] = extras_list_dictize(result, context)
#groups
member_rev = model.member_revision_table
group = model.group_table
q = select([group, member_rev.c.capacity],
from_obj=member_rev.join(group, group.c.id == member_rev.c.group_id)
).where(member_rev.c.table_id == pkg.id)\
.where(member_rev.c.state == 'active') \
.where(group.c.is_organization == False)
result = _execute_with_revision(q, member_rev, context)
context['with_capacity'] = False
## no package counts as cannot fetch from search index at the same
## time as indexing to it.
## tags, extras and sub-groups are not included for speed
result_dict["groups"] = group_list_dictize(result, context,
with_package_counts=False)
#owning organization
group_rev = model.group_revision_table
q = select([group_rev]
).where(group_rev.c.id == pkg.owner_org) \
.where(group_rev.c.state == 'active')
result = _execute_with_revision(q, group_rev, context)
organizations = d.obj_list_dictize(result, context)
if organizations:
result_dict["organization"] = organizations[0]
else:
result_dict["organization"] = None
#relations
rel_rev = model.package_relationship_revision_table
q = select([rel_rev]).where(rel_rev.c.subject_package_id == pkg.id)
result = _execute_with_revision(q, rel_rev, context)
result_dict["relationships_as_subject"] = d.obj_list_dictize(result, context)
q = select([rel_rev]).where(rel_rev.c.object_package_id == pkg.id)
result = _execute_with_revision(q, rel_rev, context)
result_dict["relationships_as_object"] = d.obj_list_dictize(result, context)
# Extra properties from the domain object
# We need an actual Package object for this, not a PackageRevision
if isinstance(pkg, model.PackageRevision):
pkg = model.Package.get(pkg.id)
# isopen
result_dict['isopen'] = pkg.isopen if isinstance(pkg.isopen,bool) else pkg.isopen()
# type
# if null assign the default value to make searching easier
result_dict['type']= pkg.type or u'dataset'
# license
if pkg.license and pkg.license.url:
result_dict['license_url']= pkg.license.url
result_dict['license_title']= pkg.license.title.split('::')[-1]
elif pkg.license:
result_dict['license_title']= pkg.license.title
else:
result_dict['license_title']= pkg.license_id
# creation and modification date
result_dict['metadata_modified'] = pkg.metadata_modified.isoformat()
result_dict['metadata_created'] = pkg.metadata_created.isoformat() \
if pkg.metadata_created else None
return result_dict
def user_following_dataset_dictize(follower, context):
return d.table_dictize(follower, context)
def related_dictize(rel, context):
return d.table_dictize(rel, context)
def package_dictize_with_revisions(pkg, context):
'''
Given a Package object, returns an equivalent dictionary.
Normally this is the most recent version, but you can provide revision_id
or revision_date in the context and it will filter to an earlier time.
May raise NotFound if:
* the specified revision_id doesn't exist
* the specified revision_date was before the package was created
'''
model = context['model']
try:
model.PackageRevision
# CKAN<=2.8
revision_model = model
except AttributeError:
# CKAN>2.8
revision_model = RevisionTableMappings.instance()
is_latest_revision = not (context.get(u'revision_id')
or context.get(u'revision_date'))
execute = _execute if is_latest_revision else _execute_with_revision
# package
if is_latest_revision:
if isinstance(pkg, revision_model.PackageRevision):
pkg = model.Package.get(pkg.id)
result = pkg
else:
package_rev = revision_model.package_revision_table
q = select([package_rev]).where(package_rev.c.id == pkg.id)
result = execute(q, package_rev, context).first()
if not result:
raise logic.NotFound
result_dict = d.table_dictize(result, context)
# strip whitespace from title
if result_dict.get(u'title'):
result_dict['title'] = result_dict['title'].strip()
# resources
if is_latest_revision:
res = model.resource_table
else:
res = revision_model.resource_revision_table
# metadata_modified was added after the revisioning was removed so
# it does not exist on the resource_revision table.
mm_col = res._columns.get(u'metadata_modified')
if mm_col is not None:
res._columns.remove(mm_col)
q = select([res]).where(res.c.package_id == pkg.id)
result = execute(q, res, context)
result_dict["resources"] = resource_list_dictize(result, context)
result_dict['num_resources'] = len(result_dict.get(u'resources', []))
# tags
tag = model.tag_table
if is_latest_revision:
pkg_tag = model.package_tag_table
else:
pkg_tag = revision_model.package_tag_revision_table
q = select([tag, pkg_tag.c.state],
from_obj=pkg_tag.join(tag, tag.c.id == pkg_tag.c.tag_id)).where(
pkg_tag.c.package_id == pkg.id)
result = execute(q, pkg_tag, context)
result_dict["tags"] = d.obj_list_dictize(result, context,
lambda x: x["name"])
result_dict['num_tags'] = len(result_dict.get(u'tags', []))
# Add display_names to tags. At first a tag's display_name is just the
# same as its name, but the display_name might get changed later (e.g.
# translated into another language by the multilingual extension).
for tag in result_dict['tags']:
assert u'display_name' not in tag
tag['display_name'] = tag['name']
# extras
if is_latest_revision:
extra = model.package_extra_table
else:
extra = revision_model.extra_revision_table
q = select([extra]).where(extra.c.package_id == pkg.id)
result = execute(q, extra, context)
result_dict["extras"] = extras_list_dictize(result, context)
# groups
if is_latest_revision:
member = model.member_table
else:
member = revision_model.member_revision_table
group = model.group_table
q = select([group, member.c.capacity],
from_obj=member.join(group, group.c.id == member.c.group_id)
).where(member.c.table_id == pkg.id)\
.where(member.c.state == u'active') \
.where(group.c.is_organization == False) # noqa
result = execute(q, member, context)
context['with_capacity'] = False
# no package counts as cannot fetch from search index at the same
# time as indexing to it.
# tags, extras and sub-groups are not included for speed
result_dict["groups"] = group_list_dictize(result,
context,
with_package_counts=False)
# owning organization
if is_latest_revision:
group = model.group_table
else:
group = revision_model.group_revision_table
q = select([group]
).where(group.c.id == result_dict['owner_org']) \
.where(group.c.state == u'active')
result = execute(q, group, context)
organizations = d.obj_list_dictize(result, context)
if organizations:
result_dict["organization"] = organizations[0]
else:
result_dict["organization"] = None
# relations
if is_latest_revision:
rel = model.package_relationship_table
else:
rel = revision_model \
.package_relationship_revision_table
q = select([rel]).where(rel.c.subject_package_id == pkg.id)
result = execute(q, rel, context)
result_dict["relationships_as_subject"] = \
d.obj_list_dictize(result, context)
q = select([rel]).where(rel.c.object_package_id == pkg.id)
result = execute(q, rel, context)
result_dict["relationships_as_object"] = \
d.obj_list_dictize(result, context)
# Extra properties from the domain object
# We need an actual Package object for this, not a PackageRevision
# if isinstance(pkg, model.PackageRevision):
# pkg = model.Package.get(pkg.id)
# isopen
result_dict['isopen'] = pkg.isopen if isinstance(pkg.isopen, bool) \
else pkg.isopen()
# type
# if null assign the default value to make searching easier
result_dict['type'] = pkg.type or u'dataset'
# license
if pkg.license and pkg.license.url:
result_dict['license_url'] = pkg.license.url
result_dict['license_title'] = pkg.license.title.split(u'::')[-1]
elif pkg.license:
result_dict['license_title'] = pkg.license.title
else:
result_dict['license_title'] = pkg.license_id
# creation and modification date
if is_latest_revision:
result_dict['metadata_modified'] = pkg.metadata_modified.isoformat()
# (If not is_latest_revision, don't use pkg which is the latest version.
# Instead, use the dates already in result_dict that came from the dictized
# PackageRevision)
result_dict['metadata_created'] = pkg.metadata_created.isoformat()
return result_dict
def _group_or_org_delete(context, data_dict, is_org=False):
'''Delete a group.
You must be authorized to delete the group.
:param id: the name or id of the group
:type id: string
'''
from sqlalchemy import or_
model = context['model']
user = context['user']
id = _get_or_bust(data_dict, 'id')
group = model.Group.get(id)
context['group'] = group
if group is None:
raise NotFound('Group was not found.')
if is_org:
_check_access('organization_delete', context, data_dict)
else:
_check_access('group_delete', context, data_dict)
# organization delete will not occur while all datasets for that org are
# not deleted
if is_org:
datasets = model.Session.query(model.Package) \
.filter_by(owner_org=group.id) \
.filter(model.Package.state != 'deleted') \
.count()
if datasets:
if not authz.check_config_permission(
'ckan.auth.create_unowned_dataset'):
raise ValidationError(
_('Organization cannot be deleted while it '
'still has datasets'))
pkg_table = model.package_table
# using Core SQLA instead of the ORM should be faster
model.Session.execute(pkg_table.update().where(
sqla.and_(
pkg_table.c.owner_org == group.id,
pkg_table.c.state != 'deleted')).values(owner_org=None))
# The group's Member objects are deleted
# (including hierarchy connections to parent and children groups)
for member in model.Session.query(model.Member).\
filter(or_(model.Member.table_id == id,
model.Member.group_id == id)).\
filter(model.Member.state == 'active').all():
member.delete()
group.delete()
if is_org:
activity_type = 'deleted organization'
else:
activity_type = 'deleted group'
activity_dict = {
'user_id': model.User.by_name(six.ensure_text(user)).id,
'object_id': group.id,
'activity_type': activity_type,
'data': {
'group': dictization.table_dictize(group, context)
}
}
activity_create_context = {
'model': model,
'defer_commit': True,
'ignore_auth': True,
'session': context['session']
}
_get_action('activity_create')(activity_create_context, activity_dict)
if is_org:
plugin_type = plugins.IOrganizationController
else:
plugin_type = plugins.IGroupController
for item in plugins.PluginImplementations(plugin_type):
item.delete(group)
model.repo.commit()
def jsonpatch_dictize(jsonpatch, context):
return d.table_dictize(jsonpatch, context)
def task_status_dictize(task_status, context):
return d.table_dictize(task_status, context)
def hdx_group_or_org_update(context, data_dict, is_org=False):
# Overriding default so that orgs can have multiple images
model = context['model']
user = context['user']
session = context['session']
id = _get_or_bust(data_dict, 'id')
group = model.Group.get(id)
context["group"] = group
if group is None:
raise NotFound('Group was not found.')
if is_org:
check_access('organization_update', context, data_dict)
else:
check_access('group_update', context, data_dict)
# get the schema
group_plugin = lib_plugins.lookup_group_plugin(group.type)
try:
schema = group_plugin.form_to_db_schema_options({'type': 'update',
'api': 'api_version' in context,
'context': context})
except AttributeError:
schema = group_plugin.form_to_db_schema()
try:
customization = json.loads(group.extras['customization'])
except:
customization = {'image_sq': '', 'image_rect': ''}
try:
data_dict['customization'] = json.loads(data_dict['customization'])
except:
data_dict['customization'] = {}
# If we're removing the image
if 'clear_image_sq' in data_dict and data_dict['clear_image_sq']:
remove_image(customization['image_sq'])
data_dict['customization']['image_sq'] = ''
customization['image_rect'] = ''
if 'clear_image_rect' in data_dict and data_dict['clear_image_rect']:
remove_image(customization['image_rect'])
data_dict['customization']['image_rect'] = ''
customization['image_rect'] = ''
if 'image_sq_upload' in data_dict and data_dict['image_sq_upload'] != '' and data_dict['image_sq_upload'] != None:
# Although weird, the FieldStorage instance has a boolean value of False so we need to compare to None
# If old image was uploaded remove it
if customization['image_sq']:
remove_image(customization['image_sq'])
upload1 = uploader.Upload('group', customization['image_sq'])
upload1.update_data_dict(data_dict, 'image_sq',
'image_sq_upload', 'clear_upload')
if 'image_rect_upload' in data_dict and data_dict['image_rect_upload'] != '' \
and data_dict['image_rect_upload'] != None:
# Although weird, the FieldStorage instance has a boolean value of False so we need to compare to None
if customization['image_rect']:
remove_image(customization['image_rect'])
upload2 = uploader.Upload('group', customization['image_rect'])
upload2.update_data_dict(data_dict, 'image_rect',
'image_rect_upload', 'clear_upload')
storage_path = uploader.get_storage_path()
##Rearrange things the way we need them
try:
if data_dict['image_sq'] != '' and data_dict['image_sq'] != None:
data_dict['customization']['image_sq'] = data_dict['image_sq']
else:
data_dict['customization']['image_sq'] = customization['image_sq']
except KeyError:
data_dict['customization']['image_sq'] = ''
try:
if data_dict['image_rect'] != '' and data_dict['image_rect'] != None:
data_dict['customization']['image_rect'] = data_dict['image_rect']
else:
data_dict['customization']['image_rect'] = customization['image_rect']
except KeyError:
data_dict['customization']['image_rect'] = ''
data_dict['customization'] = json.dumps(data_dict['customization'])
if 'api_version' not in context:
# old plugins do not support passing the schema so we need
# to ensure they still work
try:
group_plugin.check_data_dict(data_dict, schema)
except TypeError:
group_plugin.check_data_dict(data_dict)
data, errors = _validate(data_dict, schema, context)
log.debug('group_update validate_errs=%r user=%s group=%s data_dict=%r',
errors, context.get('user'),
context.get('group').name if context.get('group') else '',
data_dict)
if errors:
session.rollback()
raise ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name")
# when editing an org we do not want to update the packages if using the
# new templates.
if ((not is_org)
and not converters.asbool(
config.get('ckan.legacy_templates', False))
and 'api_version' not in context):
context['prevent_packages_update'] = True
group = model_save.group_dict_save(data, context)
if is_org:
plugin_type = plugins.IOrganizationController
else:
plugin_type = plugins.IGroupController
for item in plugins.PluginImplementations(plugin_type):
item.edit(group)
if is_org:
activity_type = 'changed organization'
else:
activity_type = 'changed group'
activity_dict = {
'user_id': model.User.by_name(user.decode('utf8')).id,
'object_id': group.id,
'activity_type': activity_type,
}
# Handle 'deleted' groups.
# When the user marks a group as deleted this comes through here as
# a 'changed' group activity. We detect this and change it to a 'deleted'
# activity.
if group.state == u'deleted':
if session.query(ckan.model.Activity).filter_by(
object_id=group.id, activity_type='deleted').all():
# A 'deleted group' activity for this group has already been
# emitted.
# FIXME: What if the group was deleted and then activated again?
activity_dict = None
else:
# We will emit a 'deleted group' activity.
activity_dict['activity_type'] = 'deleted group'
if activity_dict is not None:
activity_dict['data'] = {
'group': dictization.table_dictize(group, context)
}
activity_create_context = {
'model': model,
'user': user,
'defer_commit': True,
'ignore_auth': True,
'session': session
}
get_action('activity_create')(activity_create_context, activity_dict)
# TODO: Also create an activity detail recording what exactly changed
# in the group.
try:
upload1.upload(uploader.get_max_image_size())
except:
pass
try:
upload2.upload(uploader.get_max_image_size())
except:
pass
if not context.get('defer_commit'):
model.repo.commit()
return model_dictize.group_dictize(group, context)
def group_dictize(group, context):
result_dict = d.table_dictize(group, context)
result_dict['display_name'] = group.display_name
result_dict['extras'] = extras_dict_dictize(group._extras, context)
include_datasets = context.get('include_datasets', True)
q = {
'facet': 'false',
'rows': 0,
}
if group.is_organization:
q['fq'] = 'owner_org:"{0}"'.format(group.id)
else:
q['fq'] = 'groups:"{0}"'.format(group.name)
is_group_member = (context.get('user')
and new_authz.has_user_permission_for_group_or_org(
group.id, context.get('user'), 'read'))
if is_group_member:
context['ignore_capacity_check'] = True
if include_datasets:
q['rows'] = 1000 # Only the first 1000 datasets are returned
context_ = dict((k, v) for (k, v) in context.items() if k != 'schema')
search_results = logic.get_action('package_search')(context_, q)
if include_datasets:
result_dict['packages'] = search_results['results']
result_dict['package_count'] = search_results['count']
context['with_capacity'] = True
result_dict['tags'] = tag_list_dictize(
_get_members(context, group, 'tags'), context)
result_dict['groups'] = group_list_dictize(
_get_members(context, group, 'groups'), context)
result_dict['users'] = user_list_dictize(
_get_members(context, group, 'users'), context)
context['with_capacity'] = False
if context.get('for_view'):
if result_dict['is_organization']:
plugin = plugins.IOrganizationController
else:
plugin = plugins.IGroupController
for item in plugins.PluginImplementations(plugin):
result_dict = item.before_view(result_dict)
image_url = result_dict.get('image_url')
result_dict['image_display_url'] = image_url
if image_url and not image_url.startswith('http'):
#munge here should not have an effect only doing it incase
#of potential vulnerability of dodgy api input
image_url = munge.munge_filename(image_url)
result_dict['image_display_url'] = h.url_for_static(
'uploads/group/%s' % result_dict.get('image_url'), qualified=True)
return result_dict
def package_dictize(pkg, context):
'''
Given a Package object, returns an equivalent dictionary.
Normally this is the current revision (most recent moderated version),
but you can provide revision_id, revision_date or pending in the
context and it will filter to an earlier time or the latest unmoderated
object revision.
May raise NotFound. TODO: understand what the specific set of
circumstances are that cause this.
'''
model = context['model']
#package
package_rev = model.package_revision_table
q = select([package_rev]).where(package_rev.c.id == pkg.id)
result = _execute_with_revision(q, package_rev, context).first()
if not result:
raise NotFound
result_dict = table_dictize(result, context)
#resources
res_rev = model.resource_revision_table
resource_group = model.resource_group_table
q = select([res_rev], from_obj = res_rev.join(resource_group,
resource_group.c.id == res_rev.c.resource_group_id))
q = q.where(resource_group.c.package_id == pkg.id)
result = _execute_with_revision(q, res_rev, context)
result_dict["resources"] = resource_list_dictize(result, context)
#tags
tag_rev = model.package_tag_revision_table
tag = model.tag_table
q = select([tag, tag_rev.c.state, tag_rev.c.revision_timestamp],
from_obj=tag_rev.join(tag, tag.c.id == tag_rev.c.tag_id)
).where(tag_rev.c.package_id == pkg.id)
result = _execute_with_revision(q, tag_rev, context)
result_dict["tags"] = obj_list_dictize(result, context, lambda x: x["name"])
#extras
extra_rev = model.extra_revision_table
q = select([extra_rev]).where(extra_rev.c.package_id == pkg.id)
result = _execute_with_revision(q, extra_rev, context)
result_dict["extras"] = extras_list_dictize(result, context)
#groups
member_rev = model.member_revision_table
group = model.group_table
q = select([group],
from_obj=member_rev.join(group, group.c.id == member_rev.c.group_id)
).where(member_rev.c.table_id == pkg.id)
result = _execute_with_revision(q, member_rev, context)
result_dict["groups"] = obj_list_dictize(result, context)
#relations
rel_rev = model.package_relationship_revision_table
q = select([rel_rev]).where(rel_rev.c.subject_package_id == pkg.id)
result = _execute_with_revision(q, rel_rev, context)
result_dict["relationships_as_subject"] = obj_list_dictize(result, context)
q = select([rel_rev]).where(rel_rev.c.object_package_id == pkg.id)
result = _execute_with_revision(q, rel_rev, context)
result_dict["relationships_as_object"] = obj_list_dictize(result, context)
# Extra properties from the domain object
# We need an actual Package object for this, not a PackageRevision
if isinstance(pkg,PackageRevision):
pkg = model.Package.get(pkg.id)
# isopen
result_dict['isopen'] = pkg.isopen if isinstance(pkg.isopen,bool) else pkg.isopen()
# creation and modification date
result_dict['metadata_modified'] = pkg.metadata_modified.isoformat() \
if pkg.metadata_modified else None
result_dict['metadata_created'] = pkg.metadata_created.isoformat() \
if pkg.metadata_created else None
return result_dict
def package_dictize(pkg, context):
'''
Given a Package object, returns an equivalent dictionary.
Normally this is the current revision (most recent moderated version),
but you can provide revision_id, revision_date or pending in the
context and it will filter to an earlier time or the latest unmoderated
object revision.
May raise NotFound. TODO: understand what the specific set of
circumstances are that cause this.
'''
model = context['model']
#package
package_rev = model.package_revision_table
q = select([package_rev]).where(package_rev.c.id == pkg.id)
result = _execute_with_revision(q, package_rev, context).first()
if not result:
raise logic.NotFound
result_dict = d.table_dictize(result, context)
#resources
res_rev = model.resource_revision_table
resource_group = model.resource_group_table
q = select([res_rev], from_obj = res_rev.join(resource_group,
resource_group.c.id == res_rev.c.resource_group_id))
q = q.where(resource_group.c.package_id == pkg.id)
result = _execute_with_revision(q, res_rev, context)
result_dict["resources"] = resource_list_dictize(result, context)
#tags
tag_rev = model.package_tag_revision_table
tag = model.tag_table
q = select([tag, tag_rev.c.state, tag_rev.c.revision_timestamp],
from_obj=tag_rev.join(tag, tag.c.id == tag_rev.c.tag_id)
).where(tag_rev.c.package_id == pkg.id)
result = _execute_with_revision(q, tag_rev, context)
result_dict["tags"] = d.obj_list_dictize(result, context, lambda x: x["name"])
# Add display_names to tags. At first a tag's display_name is just the
# same as its name, but the display_name might get changed later (e.g.
# translated into another language by the multilingual extension).
for tag in result_dict['tags']:
assert not tag.has_key('display_name')
tag['display_name'] = tag['name']
#extras
extra_rev = model.extra_revision_table
q = select([extra_rev]).where(extra_rev.c.package_id == pkg.id)
result = _execute_with_revision(q, extra_rev, context)
result_dict["extras"] = extras_list_dictize(result, context)
#tracking
tracking = model.TrackingSummary.get_for_package(pkg.id)
result_dict['tracking_summary'] = tracking
#groups
member_rev = model.member_revision_table
group = model.group_table
q = select([group, member_rev.c.capacity],
from_obj=member_rev.join(group, group.c.id == member_rev.c.group_id)
).where(member_rev.c.table_id == pkg.id)\
.where(member_rev.c.state == 'active')
result = _execute_with_revision(q, member_rev, context)
result_dict["groups"] = d.obj_list_dictize(result, context)
#relations
rel_rev = model.package_relationship_revision_table
q = select([rel_rev]).where(rel_rev.c.subject_package_id == pkg.id)
result = _execute_with_revision(q, rel_rev, context)
result_dict["relationships_as_subject"] = d.obj_list_dictize(result, context)
q = select([rel_rev]).where(rel_rev.c.object_package_id == pkg.id)
result = _execute_with_revision(q, rel_rev, context)
result_dict["relationships_as_object"] = d.obj_list_dictize(result, context)
# Extra properties from the domain object
# We need an actual Package object for this, not a PackageRevision
if isinstance(pkg, ckan.model.PackageRevision):
pkg = model.Package.get(pkg.id)
# isopen
result_dict['isopen'] = pkg.isopen if isinstance(pkg.isopen,bool) else pkg.isopen()
# type
result_dict['type']= pkg.type
# licence
if pkg.license and pkg.license.url:
result_dict['license_url']= pkg.license.url
result_dict['license_title']= pkg.license.title.split('::')[-1]
elif pkg.license:
result_dict['license_title']= pkg.license.title
else:
result_dict['license_title']= pkg.license_id
# creation and modification date
result_dict['metadata_modified'] = context.pop('metadata_modified')
result_dict['metadata_created'] = pkg.metadata_created.isoformat() \
if pkg.metadata_created else None
if context.get('for_view'):
for item in plugins.PluginImplementations( plugins.IPackageController):
result_dict = item.before_view(result_dict)
return result_dict
def activity_dictize(activity, context):
activity_dict = d.table_dictize(activity, context)
return activity_dict
def group_list_dictize(obj_list,
context,
sort_key=lambda x: x['display_name'],
reverse=False,
with_package_counts=True):
active = context.get('active', True)
with_private = context.get('include_private_packages', False)
if with_package_counts:
query = search.PackageSearchQuery()
q = {
'q': '+capacity:public' if not with_private else '*:*',
'fl': 'groups',
'facet.field': ['groups', 'owner_org'],
'facet.limit': -1,
'rows': 1
}
query.run(q)
result_list = []
for obj in obj_list:
if context.get('with_capacity'):
obj, capacity = obj
group_dict = d.table_dictize(obj, context, capacity=capacity)
else:
group_dict = d.table_dictize(obj, context)
group_dict.pop('created')
if active and obj.state not in ('active', 'pending'):
continue
group_dict['display_name'] = (group_dict.get('title')
or group_dict.get('name'))
image_url = group_dict.get('image_url')
group_dict['image_display_url'] = image_url
if image_url and not image_url.startswith('http'):
#munge here should not have an effect only doing it incase
#of potential vulnerability of dodgy api input
image_url = munge.munge_filename(image_url)
group_dict['image_display_url'] = h.url_for_static(
'uploads/group/%s' % group_dict.get('image_url'),
qualified=True)
if with_package_counts:
facets = query.facets
if obj.is_organization:
group_dict['packages'] = facets['owner_org'].get(obj.id, 0)
else:
group_dict['packages'] = facets['groups'].get(obj.name, 0)
if context.get('for_view'):
if group_dict['is_organization']:
plugin = plugins.IOrganizationController
else:
plugin = plugins.IGroupController
for item in plugins.PluginImplementations(plugin):
group_dict = item.before_view(group_dict)
result_list.append(group_dict)
return sorted(result_list, key=sort_key, reverse=reverse)
def activity_detail_dictize(activity_detail, context):
return d.table_dictize(activity_detail, context)
def as_dict(self):
context = {'model': model}
oapic_dict = dictization.table_dictize(self, context)
return oapic_dict
