def current_package_list_with_resources(context, data_dict):
model = context["model"]
user = context["user"]
limit = data_dict.get("limit")
q = ckan.authz.Authorizer().authorized_query(user, model.PackageRevision)
q = q.filter(model.PackageRevision.state=='active')
q = q.filter(model.PackageRevision.current==True)
q = q.order_by(model.package_revision_table.c.revision_timestamp.desc())
if limit:
q = q.limit(limit)
pack_rev = q.all()
package_list = []
for package in pack_rev:
result_dict = table_dictize(package, context)
res_rev = model.resource_revision_table
resource_group = model.resource_group_table
q = select([res_rev], from_obj = res_rev.join(resource_group,
resource_group.c.id == res_rev.c.resource_group_id))
q = q.where(resource_group.c.package_id == package.id)
result = q.where(res_rev.c.current == True).execute()
result_dict["resources"] = resource_list_dictize(result, context)
license_id = result_dict['license_id']
if license_id:
isopen = model.Package.get_license_register()[license_id].isopen()
result_dict['isopen'] = isopen
else:
result_dict['isopen'] = False
package_list.append(result_dict)
return package_list
def _package_list_with_resources(context, package_revision_list):
package_list = []
model = context["model"]
for package in package_revision_list:
result_dict = table_dictize(package, context)
res_rev = model.resource_revision_table
resource_group = model.resource_group_table
query = select(
[res_rev], from_obj=res_rev.join(resource_group, resource_group.c.id == res_rev.c.resource_group_id)
)
query = query.where(resource_group.c.package_id == package.id)
result = query.where(res_rev.c.current == True).execute()
result_dict["resources"] = resource_list_dictize(result, context)
license_id = result_dict["license_id"]
if license_id:
try:
isopen = model.Package.get_license_register()[license_id].isopen()
result_dict["isopen"] = isopen
except KeyError:
# TODO: create a log message this error?
result_dict["isopen"] = False
else:
result_dict["isopen"] = False
package_list.append(result_dict)
return package_list
def current_package_list_with_resources(context, data_dict):
model = context["model"]
user = context["user"]
limit = data_dict.get("limit")
q = ckan.authz.Authorizer().authorized_query(user, model.PackageRevision)
q = q.filter(model.PackageRevision.state == 'active')
q = q.filter(model.PackageRevision.current == True)
q = q.order_by(model.package_revision_table.c.revision_timestamp.desc())
if limit:
q = q.limit(limit)
pack_rev = q.all()
package_list = []
for package in pack_rev:
result_dict = table_dictize(package, context)
res_rev = model.resource_revision_table
resource_group = model.resource_group_table
q = select([res_rev],
from_obj=res_rev.join(
resource_group,
resource_group.c.id == res_rev.c.resource_group_id))
q = q.where(resource_group.c.package_id == package.id)
result = q.where(res_rev.c.current == True).execute()
result_dict["resources"] = resource_list_dictize(result, context)
license_id = result_dict['license_id']
if license_id:
isopen = model.Package.get_license_register()[license_id].isopen()
result_dict['isopen'] = isopen
else:
result_dict['isopen'] = False
package_list.append(result_dict)
return package_list
def _package_list_with_resources(context, package_revision_list):
package_list = []
model = context["model"]
for package in package_revision_list:
result_dict = table_dictize(package, context)
res_rev = model.resource_revision_table
resource_group = model.resource_group_table
query = select([res_rev],
from_obj=res_rev.join(
resource_group,
resource_group.c.id == res_rev.c.resource_group_id))
query = query.where(resource_group.c.package_id == package.id)
result = query.where(res_rev.c.current == True).execute()
result_dict["resources"] = resource_list_dictize(result, context)
license_id = result_dict['license_id']
if license_id:
try:
isopen = model.Package.get_license_register(
)[license_id].isopen()
result_dict['isopen'] = isopen
except KeyError:
# TODO: create a log message this error?
result_dict['isopen'] = False
else:
result_dict['isopen'] = False
package_list.append(result_dict)
return package_list
def get_new_resources(context, data_dict):
''' Get out all new resources in this commit,
needs to be run in extension point after_create and after_update '''
model = context['model']
session = model.Session()
session.flush()
if hasattr(session, '_object_cache'):
import ckan.lib.dictization.model_dictize as model_dictize
new_objs = session._object_cache['new']
new_resources = [obj for obj in new_objs
if isinstance(obj, model.Resource)]
return model_dictize.resource_list_dictize(new_resources, context)
else:
return ()
def get_new_resources(context, data_dict):
''' Get out all new resources in this commit,
needs to be run in extension point after_create and after_update '''
model = context['model']
session = model.Session()
session.flush()
if hasattr(session, '_object_cache'):
import ckan.lib.dictization.model_dictize as model_dictize
new_objs = session._object_cache['new']
new_resources = [obj for obj in new_objs
if isinstance(obj, model.Resource)]
return model_dictize.resource_list_dictize(new_resources, context)
else:
return ()
def _extend_package_dict(package_dict,context):
model = context['model']
resources = model.Session.query(model.Resource)\
.join(model.ResourceGroup)\
.filter(model.ResourceGroup.package_id == package_dict['id'])\
.all()
if resources:
package_dict['resources'] = resource_list_dictize(resources, context)
else:
package_dict['resources'] = []
license_id = package_dict.get('license_id')
if license_id:
try:
isopen = model.Package.get_license_register()[license_id].isopen()
except KeyError:
isopen = False
package_dict['isopen'] = isopen
else:
package_dict['isopen'] = False
return package_dict
def _extend_package_dict(package_dict, context):
model = context['model']
resources = model.Session.query(model.Resource)\
.join(model.ResourceGroup)\
.filter(model.ResourceGroup.package_id == package_dict['id'])\
.all()
if resources:
package_dict['resources'] = resource_list_dictize(resources, context)
else:
package_dict['resources'] = []
license_id = package_dict.get('license_id')
if license_id:
try:
isopen = model.Package.get_license_register()[license_id].isopen()
except KeyError:
isopen = False
package_dict['isopen'] = isopen
else:
package_dict['isopen'] = False
return package_dict
def get_new_resources(context, data_dict):
'''
Returns a list of all new resources in this transaction
This is to be used only before commiting eg on `package_create` or
`package_update` as it uses a cache object stored in the Session and
filled on flushig it.
The `context` dict must contain a `model` key
Returns a list of new dictized resources.
'''
model = context['model']
session = model.Session()
session.flush()
if hasattr(session, '_object_cache'):
import ckan.lib.dictization.model_dictize as model_dictize
new_objs = session._object_cache['new']
new_resources = [obj for obj in new_objs
if isinstance(obj, model.Resource)]
return model_dictize.resource_list_dictize(new_resources, context)
else:
return []
def current_package_list_with_resources(context, data_dict):
model = context["model"]
user = context["user"]
limit = data_dict.get("limit")
check_access('current_package_list_with_resources', context, data_dict)
query = model.Session.query(model.PackageRevision)
query = query.filter(model.PackageRevision.state=='active')
query = query.filter(model.PackageRevision.current==True)
query = query.order_by(model.package_revision_table.c.revision_timestamp.desc())
if limit:
query = query.limit(limit)
pack_rev = query.all()
package_list = []
for package in pack_rev:
result_dict = table_dictize(package, context)
res_rev = model.resource_revision_table
resource_group = model.resource_group_table
query = select([res_rev], from_obj = res_rev.join(resource_group,
resource_group.c.id == res_rev.c.resource_group_id))
query = query.where(resource_group.c.package_id == package.id)
result = query.where(res_rev.c.current == True).execute()
result_dict["resources"] = resource_list_dictize(result, context)
license_id = result_dict['license_id']
if license_id:
try:
isopen = model.Package.get_license_register()[license_id].isopen()
result_dict['isopen'] = isopen
except KeyError:
# TODO: create a log message this error?
result_dict['isopen'] = False
else:
result_dict['isopen'] = False
package_list.append(result_dict)
return package_list
def _extend_package_dict(package_dict, context):
model = context["model"]
resources = (
model.Session.query(model.Resource)
.join(model.ResourceGroup)
.filter(model.ResourceGroup.package_id == package_dict["id"])
.all()
)
if resources:
package_dict["resources"] = resource_list_dictize(resources, context)
else:
package_dict["resources"] = []
license_id = package_dict.get("license_id")
if license_id:
try:
isopen = model.Package.get_license_register()[license_id].isopen()
except KeyError:
isopen = False
package_dict["isopen"] = isopen
else:
package_dict["isopen"] = False
return package_dict
def package_dictize_with_revisions(pkg, context):
'''
Given a Package object, returns an equivalent dictionary.
Normally this is the most recent version, but you can provide revision_id
or revision_date in the context and it will filter to an earlier time.
May raise NotFound if:
* the specified revision_id doesn't exist
* the specified revision_date was before the package was created
'''
model = context['model']
try:
model.PackageRevision
# CKAN<=2.8
revision_model = model
except AttributeError:
# CKAN>2.8
revision_model = RevisionTableMappings.instance()
is_latest_revision = not(context.get(u'revision_id') or
context.get(u'revision_date'))
execute = _execute if is_latest_revision else _execute_with_revision
# package
if is_latest_revision:
if isinstance(pkg, revision_model.PackageRevision):
pkg = model.Package.get(pkg.id)
result = pkg
else:
package_rev = revision_model.package_revision_table
q = select([package_rev]).where(package_rev.c.id == pkg.id)
result = execute(q, package_rev, context).first()
if not result:
raise logic.NotFound
result_dict = d.table_dictize(result, context)
# strip whitespace from title
if result_dict.get(u'title'):
result_dict['title'] = result_dict['title'].strip()
# resources
if is_latest_revision:
res = model.resource_table
else:
res = revision_model.resource_revision_table
# metadata_modified was added after the revisioning was removed so
# it does not exist on the resource_revision table.
mm_col = res._columns.get(u'metadata_modified')
if mm_col is not None:
res._columns.remove(mm_col)
q = select([res]).where(res.c.package_id == pkg.id)
result = execute(q, res, context)
result_dict["resources"] = resource_list_dictize(result, context)
result_dict['num_resources'] = len(result_dict.get(u'resources', []))
# tags
tag = model.tag_table
if is_latest_revision:
pkg_tag = model.package_tag_table
else:
pkg_tag = revision_model.package_tag_revision_table
q = select([tag, pkg_tag.c.state],
from_obj=pkg_tag.join(tag, tag.c.id == pkg_tag.c.tag_id)
).where(pkg_tag.c.package_id == pkg.id)
result = execute(q, pkg_tag, context)
result_dict["tags"] = d.obj_list_dictize(result, context,
lambda x: x["name"])
result_dict['num_tags'] = len(result_dict.get(u'tags', []))
# Add display_names to tags. At first a tag's display_name is just the
# same as its name, but the display_name might get changed later (e.g.
# translated into another language by the multilingual extension).
for tag in result_dict['tags']:
assert u'display_name' not in tag
tag['display_name'] = tag['name']
# extras
if is_latest_revision:
extra = model.package_extra_table
else:
extra = revision_model.extra_revision_table
q = select([extra]).where(extra.c.package_id == pkg.id)
result = execute(q, extra, context)
result_dict["extras"] = extras_list_dictize(result, context)
# groups
if is_latest_revision:
member = model.member_table
else:
member = revision_model.member_revision_table
group = model.group_table
q = select([group, member.c.capacity],
from_obj=member.join(group, group.c.id == member.c.group_id)
).where(member.c.table_id == pkg.id)\
.where(member.c.state == u'active') \
.where(group.c.is_organization == False) # noqa
result = execute(q, member, context)
context['with_capacity'] = False
# no package counts as cannot fetch from search index at the same
# time as indexing to it.
# tags, extras and sub-groups are not included for speed
result_dict["groups"] = group_list_dictize(result, context,
with_package_counts=False)
# owning organization
if is_latest_revision:
group = model.group_table
else:
group = revision_model.group_revision_table
q = select([group]
).where(group.c.id == result_dict['owner_org']) \
.where(group.c.state == u'active')
result = execute(q, group, context)
organizations = d.obj_list_dictize(result, context)
if organizations:
result_dict["organization"] = organizations[0]
else:
result_dict["organization"] = None
# relations
if is_latest_revision:
rel = model.package_relationship_table
else:
rel = revision_model \
.package_relationship_revision_table
q = select([rel]).where(rel.c.subject_package_id == pkg.id)
result = execute(q, rel, context)
result_dict["relationships_as_subject"] = \
d.obj_list_dictize(result, context)
q = select([rel]).where(rel.c.object_package_id == pkg.id)
result = execute(q, rel, context)
result_dict["relationships_as_object"] = \
d.obj_list_dictize(result, context)
# Extra properties from the domain object
# We need an actual Package object for this, not a PackageRevision
# if isinstance(pkg, model.PackageRevision):
# pkg = model.Package.get(pkg.id)
# isopen
result_dict['isopen'] = pkg.isopen if isinstance(pkg.isopen, bool) \
else pkg.isopen()
# type
# if null assign the default value to make searching easier
result_dict['type'] = pkg.type or u'dataset'
# license
if pkg.license and pkg.license.url:
result_dict['license_url'] = pkg.license.url
result_dict['license_title'] = pkg.license.title.split(u'::')[-1]
elif pkg.license:
result_dict['license_title'] = pkg.license.title
else:
result_dict['license_title'] = pkg.license_id
# creation and modification date
if is_latest_revision:
result_dict['metadata_modified'] = pkg.metadata_modified.isoformat()
# (If not is_latest_revision, don't use pkg which is the latest version.
# Instead, use the dates already in result_dict that came from the dictized
# PackageRevision)
result_dict['metadata_created'] = pkg.metadata_created.isoformat()
return result_dict
def package_dictize_with_revisions(pkg, context):
'''
Given a Package object, returns an equivalent dictionary.
Normally this is the most recent version, but you can provide revision_id
or revision_date in the context and it will filter to an earlier time.
May raise NotFound if:
* the specified revision_id doesn't exist
* the specified revision_date was before the package was created
'''
model = context['model']
is_latest_revision = not(context.get(u'revision_id') or
context.get(u'revision_date'))
execute = _execute if is_latest_revision else _execute_with_revision
# package
if is_latest_revision:
if isinstance(pkg, revision_model.PackageRevision):
pkg = model.Package.get(pkg.id)
result = pkg
else:
package_rev = revision_model.package_revision_table
q = select([package_rev]).where(package_rev.c.id == pkg.id)
result = execute(q, package_rev, context).first()
if not result:
raise logic.NotFound
result_dict = d.table_dictize(result, context)
# strip whitespace from title
if result_dict.get(u'title'):
result_dict['title'] = result_dict['title'].strip()
# resources
if is_latest_revision:
res = model.resource_table
else:
res = revision_model.resource_revision_table
q = select([res]).where(res.c.package_id == pkg.id)
result = execute(q, res, context)
result_dict["resources"] = resource_list_dictize(result, context)
result_dict['num_resources'] = len(result_dict.get(u'resources', []))
# tags
tag = model.tag_table
if is_latest_revision:
pkg_tag = model.package_tag_table
else:
pkg_tag = revision_model.package_tag_revision_table
q = select([tag, pkg_tag.c.state],
from_obj=pkg_tag.join(tag, tag.c.id == pkg_tag.c.tag_id)
).where(pkg_tag.c.package_id == pkg.id)
result = execute(q, pkg_tag, context)
result_dict["tags"] = d.obj_list_dictize(result, context,
lambda x: x["name"])
result_dict['num_tags'] = len(result_dict.get(u'tags', []))
# Add display_names to tags. At first a tag's display_name is just the
# same as its name, but the display_name might get changed later (e.g.
# translated into another language by the multilingual extension).
for tag in result_dict['tags']:
assert u'display_name' not in tag
tag['display_name'] = tag['name']
# extras
if is_latest_revision:
extra = model.package_extra_table
else:
extra = revision_model.extra_revision_table
q = select([extra]).where(extra.c.package_id == pkg.id)
result = execute(q, extra, context)
result_dict["extras"] = extras_list_dictize(result, context)
# groups
if is_latest_revision:
member = model.member_table
else:
member = revision_model.member_revision_table
group = model.group_table
q = select([group, member.c.capacity],
from_obj=member.join(group, group.c.id == member.c.group_id)
).where(member.c.table_id == pkg.id)\
.where(member.c.state == u'active') \
.where(group.c.is_organization == False) # noqa
result = execute(q, member, context)
context['with_capacity'] = False
# no package counts as cannot fetch from search index at the same
# time as indexing to it.
# tags, extras and sub-groups are not included for speed
result_dict["groups"] = group_list_dictize(result, context,
with_package_counts=False)
# owning organization
if is_latest_revision:
group = model.group_table
else:
group = revision_model.group_revision_table
q = select([group]
).where(group.c.id == result_dict['owner_org']) \
.where(group.c.state == u'active')
result = execute(q, group, context)
organizations = d.obj_list_dictize(result, context)
if organizations:
result_dict["organization"] = organizations[0]
else:
result_dict["organization"] = None
# relations
if is_latest_revision:
rel = model.package_relationship_table
else:
rel = revision_model \
.package_relationship_revision_table
q = select([rel]).where(rel.c.subject_package_id == pkg.id)
result = execute(q, rel, context)
result_dict["relationships_as_subject"] = \
d.obj_list_dictize(result, context)
q = select([rel]).where(rel.c.object_package_id == pkg.id)
result = execute(q, rel, context)
result_dict["relationships_as_object"] = \
d.obj_list_dictize(result, context)
# Extra properties from the domain object
# We need an actual Package object for this, not a PackageRevision
# if isinstance(pkg, model.PackageRevision):
# pkg = model.Package.get(pkg.id)
# isopen
result_dict['isopen'] = pkg.isopen if isinstance(pkg.isopen, bool) \
else pkg.isopen()
# type
# if null assign the default value to make searching easier
result_dict['type'] = pkg.type or u'dataset'
# license
if pkg.license and pkg.license.url:
result_dict['license_url'] = pkg.license.url
result_dict['license_title'] = pkg.license.title.split(u'::')[-1]
elif pkg.license:
result_dict['license_title'] = pkg.license.title
else:
result_dict['license_title'] = pkg.license_id
# creation and modification date
if is_latest_revision:
result_dict['metadata_modified'] = pkg.metadata_modified.isoformat()
# (If not is_latest_revision, don't use pkg which is the latest version.
# Instead, use the dates already in result_dict that came from the dictized
# PackageRevision)
result_dict['metadata_created'] = pkg.metadata_created.isoformat()
return result_dict
def resource_search(context, data_dict):
model = context['model']
query = data_dict.get('query')
fields = data_dict.get('fields')
if query is None and fields is None:
raise ValidationError({'query': _('Missing value')})
elif query is not None and fields is not None:
raise ValidationError(
{'fields': _('Do not specify if using "query" parameter')})
elif query is not None:
if isinstance(query, basestring):
query = [query]
try:
fields = dict(pair.split(":", 1) for pair in query)
except ValueError:
raise ValidationError(
{'query': _('Must be <field>:<value> pair(s)')})
else:
log.warning('Use of the "fields" parameter in resource_search is '
'deprecated. Use the "query" parameter instead')
# The legacy fields paramter splits string terms.
# So maintain that behaviour
split_terms = {}
for field, terms in fields.items():
if isinstance(terms, basestring):
terms = terms.split()
split_terms[field] = terms
fields = split_terms
order_by = data_dict.get('order_by')
offset = data_dict.get('offset')
limit = data_dict.get('limit')
q = model.Session.query(model.Resource) \
.join(model.Package) \
.filter(model.Package.state == 'active') \
.filter(model.Package.private == False) \
.filter(model.Resource.state == 'active') \
resource_fields = model.Resource.get_columns()
for field, terms in fields.items():
if isinstance(terms, basestring):
terms = [terms]
if field not in resource_fields:
msg = _('Field "{field}" not recognised in resource_search.')\
.format(field=field)
# Running in the context of the internal search api.
if context.get('search_query', False):
raise search.SearchError(msg)
# Otherwise, assume we're in the context of an external api
# and need to provide meaningful external error messages.
raise ValidationError({'query': msg})
for term in terms:
# prevent pattern injection
term = misc.escape_sql_like_special_characters(term)
model_attr = getattr(model.Resource, field)
# Treat the has field separately, see docstring.
if field == 'hash':
q = q.filter(model_attr.ilike(unicode(term) + '%'))
# Resource extras are stored in a json blob. So searching for
# matching fields is a bit trickier. See the docstring.
elif field in model.Resource.get_extra_columns():
model_attr = getattr(model.Resource, 'extras')
like = _or_(
model_attr.ilike(
u'''%%"%s": "%%%s%%",%%''' % (field, term)),
model_attr.ilike(
u'''%%"%s": "%%%s%%"}''' % (field, term))
)
q = q.filter(like)
# Just a regular field
else:
q = q.filter(model_attr.ilike('%' + unicode(term) + '%'))
if order_by is not None:
if hasattr(model.Resource, order_by):
q = q.order_by(getattr(model.Resource, order_by))
count = q.count()
q = q.offset(offset)
q = q.limit(limit)
results = []
for result in q:
if isinstance(result, tuple) \
and isinstance(result[0], model.DomainObject):
# This is the case for order_by rank due to the add_column.
if result[0].package.extras.get("restricted", 0) == "1" and not context.get("user", None):
continue
results.append(result[0])
else:
if result.package.extras.get("restricted", 0) == "1" and not context.get("user", None):
continue
results.append(result)
# If run in the context of a search query, then don't dictize the results.
if not context.get('search_query', False):
results = model_dictize.resource_list_dictize(results, context)
return {'count': count,
'results': results}