def current_package_list_with_resources(context, data_dict): model = context["model"] user = context["user"] limit = data_dict.get("limit") q = ckan.authz.Authorizer().authorized_query(user, model.PackageRevision) q = q.filter(model.PackageRevision.state=='active') q = q.filter(model.PackageRevision.current==True) q = q.order_by(model.package_revision_table.c.revision_timestamp.desc()) if limit: q = q.limit(limit) pack_rev = q.all() package_list = [] for package in pack_rev: result_dict = table_dictize(package, context) res_rev = model.resource_revision_table resource_group = model.resource_group_table q = select([res_rev], from_obj = res_rev.join(resource_group, resource_group.c.id == res_rev.c.resource_group_id)) q = q.where(resource_group.c.package_id == package.id) result = q.where(res_rev.c.current == True).execute() result_dict["resources"] = resource_list_dictize(result, context) license_id = result_dict['license_id'] if license_id: isopen = model.Package.get_license_register()[license_id].isopen() result_dict['isopen'] = isopen else: result_dict['isopen'] = False package_list.append(result_dict) return package_list
def _package_list_with_resources(context, package_revision_list): package_list = [] model = context["model"] for package in package_revision_list: result_dict = table_dictize(package, context) res_rev = model.resource_revision_table resource_group = model.resource_group_table query = select( [res_rev], from_obj=res_rev.join(resource_group, resource_group.c.id == res_rev.c.resource_group_id) ) query = query.where(resource_group.c.package_id == package.id) result = query.where(res_rev.c.current == True).execute() result_dict["resources"] = resource_list_dictize(result, context) license_id = result_dict["license_id"] if license_id: try: isopen = model.Package.get_license_register()[license_id].isopen() result_dict["isopen"] = isopen except KeyError: # TODO: create a log message this error? result_dict["isopen"] = False else: result_dict["isopen"] = False package_list.append(result_dict) return package_list
def current_package_list_with_resources(context, data_dict): model = context["model"] user = context["user"] limit = data_dict.get("limit") q = ckan.authz.Authorizer().authorized_query(user, model.PackageRevision) q = q.filter(model.PackageRevision.state == 'active') q = q.filter(model.PackageRevision.current == True) q = q.order_by(model.package_revision_table.c.revision_timestamp.desc()) if limit: q = q.limit(limit) pack_rev = q.all() package_list = [] for package in pack_rev: result_dict = table_dictize(package, context) res_rev = model.resource_revision_table resource_group = model.resource_group_table q = select([res_rev], from_obj=res_rev.join( resource_group, resource_group.c.id == res_rev.c.resource_group_id)) q = q.where(resource_group.c.package_id == package.id) result = q.where(res_rev.c.current == True).execute() result_dict["resources"] = resource_list_dictize(result, context) license_id = result_dict['license_id'] if license_id: isopen = model.Package.get_license_register()[license_id].isopen() result_dict['isopen'] = isopen else: result_dict['isopen'] = False package_list.append(result_dict) return package_list
def _package_list_with_resources(context, package_revision_list): package_list = [] model = context["model"] for package in package_revision_list: result_dict = table_dictize(package, context) res_rev = model.resource_revision_table resource_group = model.resource_group_table query = select([res_rev], from_obj=res_rev.join( resource_group, resource_group.c.id == res_rev.c.resource_group_id)) query = query.where(resource_group.c.package_id == package.id) result = query.where(res_rev.c.current == True).execute() result_dict["resources"] = resource_list_dictize(result, context) license_id = result_dict['license_id'] if license_id: try: isopen = model.Package.get_license_register( )[license_id].isopen() result_dict['isopen'] = isopen except KeyError: # TODO: create a log message this error? result_dict['isopen'] = False else: result_dict['isopen'] = False package_list.append(result_dict) return package_list
def get_new_resources(context, data_dict): ''' Get out all new resources in this commit, needs to be run in extension point after_create and after_update ''' model = context['model'] session = model.Session() session.flush() if hasattr(session, '_object_cache'): import ckan.lib.dictization.model_dictize as model_dictize new_objs = session._object_cache['new'] new_resources = [obj for obj in new_objs if isinstance(obj, model.Resource)] return model_dictize.resource_list_dictize(new_resources, context) else: return ()
def _extend_package_dict(package_dict,context): model = context['model'] resources = model.Session.query(model.Resource)\ .join(model.ResourceGroup)\ .filter(model.ResourceGroup.package_id == package_dict['id'])\ .all() if resources: package_dict['resources'] = resource_list_dictize(resources, context) else: package_dict['resources'] = [] license_id = package_dict.get('license_id') if license_id: try: isopen = model.Package.get_license_register()[license_id].isopen() except KeyError: isopen = False package_dict['isopen'] = isopen else: package_dict['isopen'] = False return package_dict
def _extend_package_dict(package_dict, context): model = context['model'] resources = model.Session.query(model.Resource)\ .join(model.ResourceGroup)\ .filter(model.ResourceGroup.package_id == package_dict['id'])\ .all() if resources: package_dict['resources'] = resource_list_dictize(resources, context) else: package_dict['resources'] = [] license_id = package_dict.get('license_id') if license_id: try: isopen = model.Package.get_license_register()[license_id].isopen() except KeyError: isopen = False package_dict['isopen'] = isopen else: package_dict['isopen'] = False return package_dict
def get_new_resources(context, data_dict): ''' Returns a list of all new resources in this transaction This is to be used only before commiting eg on `package_create` or `package_update` as it uses a cache object stored in the Session and filled on flushig it. The `context` dict must contain a `model` key Returns a list of new dictized resources. ''' model = context['model'] session = model.Session() session.flush() if hasattr(session, '_object_cache'): import ckan.lib.dictization.model_dictize as model_dictize new_objs = session._object_cache['new'] new_resources = [obj for obj in new_objs if isinstance(obj, model.Resource)] return model_dictize.resource_list_dictize(new_resources, context) else: return []
def current_package_list_with_resources(context, data_dict): model = context["model"] user = context["user"] limit = data_dict.get("limit") check_access('current_package_list_with_resources', context, data_dict) query = model.Session.query(model.PackageRevision) query = query.filter(model.PackageRevision.state=='active') query = query.filter(model.PackageRevision.current==True) query = query.order_by(model.package_revision_table.c.revision_timestamp.desc()) if limit: query = query.limit(limit) pack_rev = query.all() package_list = [] for package in pack_rev: result_dict = table_dictize(package, context) res_rev = model.resource_revision_table resource_group = model.resource_group_table query = select([res_rev], from_obj = res_rev.join(resource_group, resource_group.c.id == res_rev.c.resource_group_id)) query = query.where(resource_group.c.package_id == package.id) result = query.where(res_rev.c.current == True).execute() result_dict["resources"] = resource_list_dictize(result, context) license_id = result_dict['license_id'] if license_id: try: isopen = model.Package.get_license_register()[license_id].isopen() result_dict['isopen'] = isopen except KeyError: # TODO: create a log message this error? result_dict['isopen'] = False else: result_dict['isopen'] = False package_list.append(result_dict) return package_list
def _extend_package_dict(package_dict, context): model = context["model"] resources = ( model.Session.query(model.Resource) .join(model.ResourceGroup) .filter(model.ResourceGroup.package_id == package_dict["id"]) .all() ) if resources: package_dict["resources"] = resource_list_dictize(resources, context) else: package_dict["resources"] = [] license_id = package_dict.get("license_id") if license_id: try: isopen = model.Package.get_license_register()[license_id].isopen() except KeyError: isopen = False package_dict["isopen"] = isopen else: package_dict["isopen"] = False return package_dict
def package_dictize_with_revisions(pkg, context): ''' Given a Package object, returns an equivalent dictionary. Normally this is the most recent version, but you can provide revision_id or revision_date in the context and it will filter to an earlier time. May raise NotFound if: * the specified revision_id doesn't exist * the specified revision_date was before the package was created ''' model = context['model'] try: model.PackageRevision # CKAN<=2.8 revision_model = model except AttributeError: # CKAN>2.8 revision_model = RevisionTableMappings.instance() is_latest_revision = not(context.get(u'revision_id') or context.get(u'revision_date')) execute = _execute if is_latest_revision else _execute_with_revision # package if is_latest_revision: if isinstance(pkg, revision_model.PackageRevision): pkg = model.Package.get(pkg.id) result = pkg else: package_rev = revision_model.package_revision_table q = select([package_rev]).where(package_rev.c.id == pkg.id) result = execute(q, package_rev, context).first() if not result: raise logic.NotFound result_dict = d.table_dictize(result, context) # strip whitespace from title if result_dict.get(u'title'): result_dict['title'] = result_dict['title'].strip() # resources if is_latest_revision: res = model.resource_table else: res = revision_model.resource_revision_table # metadata_modified was added after the revisioning was removed so # it does not exist on the resource_revision table. mm_col = res._columns.get(u'metadata_modified') if mm_col is not None: res._columns.remove(mm_col) q = select([res]).where(res.c.package_id == pkg.id) result = execute(q, res, context) result_dict["resources"] = resource_list_dictize(result, context) result_dict['num_resources'] = len(result_dict.get(u'resources', [])) # tags tag = model.tag_table if is_latest_revision: pkg_tag = model.package_tag_table else: pkg_tag = revision_model.package_tag_revision_table q = select([tag, pkg_tag.c.state], from_obj=pkg_tag.join(tag, tag.c.id == pkg_tag.c.tag_id) ).where(pkg_tag.c.package_id == pkg.id) result = execute(q, pkg_tag, context) result_dict["tags"] = d.obj_list_dictize(result, context, lambda x: x["name"]) result_dict['num_tags'] = len(result_dict.get(u'tags', [])) # Add display_names to tags. At first a tag's display_name is just the # same as its name, but the display_name might get changed later (e.g. # translated into another language by the multilingual extension). for tag in result_dict['tags']: assert u'display_name' not in tag tag['display_name'] = tag['name'] # extras if is_latest_revision: extra = model.package_extra_table else: extra = revision_model.extra_revision_table q = select([extra]).where(extra.c.package_id == pkg.id) result = execute(q, extra, context) result_dict["extras"] = extras_list_dictize(result, context) # groups if is_latest_revision: member = model.member_table else: member = revision_model.member_revision_table group = model.group_table q = select([group, member.c.capacity], from_obj=member.join(group, group.c.id == member.c.group_id) ).where(member.c.table_id == pkg.id)\ .where(member.c.state == u'active') \ .where(group.c.is_organization == False) # noqa result = execute(q, member, context) context['with_capacity'] = False # no package counts as cannot fetch from search index at the same # time as indexing to it. # tags, extras and sub-groups are not included for speed result_dict["groups"] = group_list_dictize(result, context, with_package_counts=False) # owning organization if is_latest_revision: group = model.group_table else: group = revision_model.group_revision_table q = select([group] ).where(group.c.id == result_dict['owner_org']) \ .where(group.c.state == u'active') result = execute(q, group, context) organizations = d.obj_list_dictize(result, context) if organizations: result_dict["organization"] = organizations[0] else: result_dict["organization"] = None # relations if is_latest_revision: rel = model.package_relationship_table else: rel = revision_model \ .package_relationship_revision_table q = select([rel]).where(rel.c.subject_package_id == pkg.id) result = execute(q, rel, context) result_dict["relationships_as_subject"] = \ d.obj_list_dictize(result, context) q = select([rel]).where(rel.c.object_package_id == pkg.id) result = execute(q, rel, context) result_dict["relationships_as_object"] = \ d.obj_list_dictize(result, context) # Extra properties from the domain object # We need an actual Package object for this, not a PackageRevision # if isinstance(pkg, model.PackageRevision): # pkg = model.Package.get(pkg.id) # isopen result_dict['isopen'] = pkg.isopen if isinstance(pkg.isopen, bool) \ else pkg.isopen() # type # if null assign the default value to make searching easier result_dict['type'] = pkg.type or u'dataset' # license if pkg.license and pkg.license.url: result_dict['license_url'] = pkg.license.url result_dict['license_title'] = pkg.license.title.split(u'::')[-1] elif pkg.license: result_dict['license_title'] = pkg.license.title else: result_dict['license_title'] = pkg.license_id # creation and modification date if is_latest_revision: result_dict['metadata_modified'] = pkg.metadata_modified.isoformat() # (If not is_latest_revision, don't use pkg which is the latest version. # Instead, use the dates already in result_dict that came from the dictized # PackageRevision) result_dict['metadata_created'] = pkg.metadata_created.isoformat() return result_dict
def package_dictize_with_revisions(pkg, context): ''' Given a Package object, returns an equivalent dictionary. Normally this is the most recent version, but you can provide revision_id or revision_date in the context and it will filter to an earlier time. May raise NotFound if: * the specified revision_id doesn't exist * the specified revision_date was before the package was created ''' model = context['model'] is_latest_revision = not(context.get(u'revision_id') or context.get(u'revision_date')) execute = _execute if is_latest_revision else _execute_with_revision # package if is_latest_revision: if isinstance(pkg, revision_model.PackageRevision): pkg = model.Package.get(pkg.id) result = pkg else: package_rev = revision_model.package_revision_table q = select([package_rev]).where(package_rev.c.id == pkg.id) result = execute(q, package_rev, context).first() if not result: raise logic.NotFound result_dict = d.table_dictize(result, context) # strip whitespace from title if result_dict.get(u'title'): result_dict['title'] = result_dict['title'].strip() # resources if is_latest_revision: res = model.resource_table else: res = revision_model.resource_revision_table q = select([res]).where(res.c.package_id == pkg.id) result = execute(q, res, context) result_dict["resources"] = resource_list_dictize(result, context) result_dict['num_resources'] = len(result_dict.get(u'resources', [])) # tags tag = model.tag_table if is_latest_revision: pkg_tag = model.package_tag_table else: pkg_tag = revision_model.package_tag_revision_table q = select([tag, pkg_tag.c.state], from_obj=pkg_tag.join(tag, tag.c.id == pkg_tag.c.tag_id) ).where(pkg_tag.c.package_id == pkg.id) result = execute(q, pkg_tag, context) result_dict["tags"] = d.obj_list_dictize(result, context, lambda x: x["name"]) result_dict['num_tags'] = len(result_dict.get(u'tags', [])) # Add display_names to tags. At first a tag's display_name is just the # same as its name, but the display_name might get changed later (e.g. # translated into another language by the multilingual extension). for tag in result_dict['tags']: assert u'display_name' not in tag tag['display_name'] = tag['name'] # extras if is_latest_revision: extra = model.package_extra_table else: extra = revision_model.extra_revision_table q = select([extra]).where(extra.c.package_id == pkg.id) result = execute(q, extra, context) result_dict["extras"] = extras_list_dictize(result, context) # groups if is_latest_revision: member = model.member_table else: member = revision_model.member_revision_table group = model.group_table q = select([group, member.c.capacity], from_obj=member.join(group, group.c.id == member.c.group_id) ).where(member.c.table_id == pkg.id)\ .where(member.c.state == u'active') \ .where(group.c.is_organization == False) # noqa result = execute(q, member, context) context['with_capacity'] = False # no package counts as cannot fetch from search index at the same # time as indexing to it. # tags, extras and sub-groups are not included for speed result_dict["groups"] = group_list_dictize(result, context, with_package_counts=False) # owning organization if is_latest_revision: group = model.group_table else: group = revision_model.group_revision_table q = select([group] ).where(group.c.id == result_dict['owner_org']) \ .where(group.c.state == u'active') result = execute(q, group, context) organizations = d.obj_list_dictize(result, context) if organizations: result_dict["organization"] = organizations[0] else: result_dict["organization"] = None # relations if is_latest_revision: rel = model.package_relationship_table else: rel = revision_model \ .package_relationship_revision_table q = select([rel]).where(rel.c.subject_package_id == pkg.id) result = execute(q, rel, context) result_dict["relationships_as_subject"] = \ d.obj_list_dictize(result, context) q = select([rel]).where(rel.c.object_package_id == pkg.id) result = execute(q, rel, context) result_dict["relationships_as_object"] = \ d.obj_list_dictize(result, context) # Extra properties from the domain object # We need an actual Package object for this, not a PackageRevision # if isinstance(pkg, model.PackageRevision): # pkg = model.Package.get(pkg.id) # isopen result_dict['isopen'] = pkg.isopen if isinstance(pkg.isopen, bool) \ else pkg.isopen() # type # if null assign the default value to make searching easier result_dict['type'] = pkg.type or u'dataset' # license if pkg.license and pkg.license.url: result_dict['license_url'] = pkg.license.url result_dict['license_title'] = pkg.license.title.split(u'::')[-1] elif pkg.license: result_dict['license_title'] = pkg.license.title else: result_dict['license_title'] = pkg.license_id # creation and modification date if is_latest_revision: result_dict['metadata_modified'] = pkg.metadata_modified.isoformat() # (If not is_latest_revision, don't use pkg which is the latest version. # Instead, use the dates already in result_dict that came from the dictized # PackageRevision) result_dict['metadata_created'] = pkg.metadata_created.isoformat() return result_dict
def resource_search(context, data_dict): model = context['model'] query = data_dict.get('query') fields = data_dict.get('fields') if query is None and fields is None: raise ValidationError({'query': _('Missing value')}) elif query is not None and fields is not None: raise ValidationError( {'fields': _('Do not specify if using "query" parameter')}) elif query is not None: if isinstance(query, basestring): query = [query] try: fields = dict(pair.split(":", 1) for pair in query) except ValueError: raise ValidationError( {'query': _('Must be <field>:<value> pair(s)')}) else: log.warning('Use of the "fields" parameter in resource_search is ' 'deprecated. Use the "query" parameter instead') # The legacy fields paramter splits string terms. # So maintain that behaviour split_terms = {} for field, terms in fields.items(): if isinstance(terms, basestring): terms = terms.split() split_terms[field] = terms fields = split_terms order_by = data_dict.get('order_by') offset = data_dict.get('offset') limit = data_dict.get('limit') q = model.Session.query(model.Resource) \ .join(model.Package) \ .filter(model.Package.state == 'active') \ .filter(model.Package.private == False) \ .filter(model.Resource.state == 'active') \ resource_fields = model.Resource.get_columns() for field, terms in fields.items(): if isinstance(terms, basestring): terms = [terms] if field not in resource_fields: msg = _('Field "{field}" not recognised in resource_search.')\ .format(field=field) # Running in the context of the internal search api. if context.get('search_query', False): raise search.SearchError(msg) # Otherwise, assume we're in the context of an external api # and need to provide meaningful external error messages. raise ValidationError({'query': msg}) for term in terms: # prevent pattern injection term = misc.escape_sql_like_special_characters(term) model_attr = getattr(model.Resource, field) # Treat the has field separately, see docstring. if field == 'hash': q = q.filter(model_attr.ilike(unicode(term) + '%')) # Resource extras are stored in a json blob. So searching for # matching fields is a bit trickier. See the docstring. elif field in model.Resource.get_extra_columns(): model_attr = getattr(model.Resource, 'extras') like = _or_( model_attr.ilike( u'''%%"%s": "%%%s%%",%%''' % (field, term)), model_attr.ilike( u'''%%"%s": "%%%s%%"}''' % (field, term)) ) q = q.filter(like) # Just a regular field else: q = q.filter(model_attr.ilike('%' + unicode(term) + '%')) if order_by is not None: if hasattr(model.Resource, order_by): q = q.order_by(getattr(model.Resource, order_by)) count = q.count() q = q.offset(offset) q = q.limit(limit) results = [] for result in q: if isinstance(result, tuple) \ and isinstance(result[0], model.DomainObject): # This is the case for order_by rank due to the add_column. if result[0].package.extras.get("restricted", 0) == "1" and not context.get("user", None): continue results.append(result[0]) else: if result.package.extras.get("restricted", 0) == "1" and not context.get("user", None): continue results.append(result) # If run in the context of a search query, then don't dictize the results. if not context.get('search_query', False): results = model_dictize.resource_list_dictize(results, context) return {'count': count, 'results': results}