def test_08_package_save(self): context = {"model": model, "user": '******', "session": model.Session} anna1 = model.Session.query(model.Package).filter_by(name='annakarenina').one() anna_dictized = self.remove_changable_columns(package_dictize(anna1, context)) anna_dictized["name"] = u'annakarenina3' model.repo.new_revision() package_dict_save(anna_dictized, context) model.Session.commit() # Re-clean anna_dictized anna_dictized = self.remove_changable_columns(anna_dictized) pkg = model.Session.query(model.Package).filter_by(name='annakarenina3').one() package_dictized = self.remove_changable_columns(package_dictize(pkg, context)) anna_original = pformat(anna_dictized) anna_after_save = pformat(package_dictized) assert package_dictized == anna_dictized,\ "\n".join(unified_diff(anna_original.split("\n"), anna_after_save.split("\n")))
def test_13_get_package_in_past(self): context = {'model': model, 'session': model.Session} anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed2').one() pkgrevisions = model.Session.query(model.PackageRevision).filter_by(id=anna1.id).all() sorted_packages = sorted(pkgrevisions, key=lambda x:x.revision_timestamp) context['revision_id'] = sorted_packages[0].revision_id #original state first_dictized = self.remove_changable_columns(package_dictize(anna1, context)) assert self.package_expected == first_dictized context['revision_id'] = sorted_packages[1].revision_id #original state second_dictized = self.remove_changable_columns(package_dictize(anna1, context)) first_dictized["name"] = u'annakarenina_changed' first_dictized["resources"][0]["url"] = u'new_url' assert second_dictized == first_dictized context['revision_id'] = sorted_packages[2].revision_id #original state third_dictized = self.remove_changable_columns(package_dictize(anna1, context)) second_dictized['name'] = u'annakarenina_changed2' second_dictized['resources'][0]['url'] = u'new_url2' second_dictized['tags'][0]['name'] = u'new_tag' second_dictized['extras'][0]['value'] = u'"new_value"' second_dictized['state'] = 'pending' assert second_dictized == third_dictized context['revision_id'] = sorted_packages[3].revision_id #original state forth_dictized = self.remove_changable_columns(package_dictize(anna1, context)) third_dictized['notes'] = 'wee' third_dictized['resources'].insert(2, {u'description': u'', u'format': u'plain text', u'hash': u'', u'position': 2, u'state': u'active', u'url': u'newurl'}) third_dictized['tags'].insert(1, {'name': u'newnew_tag', 'state': 'active'}) third_dictized['extras'].insert(0, {'key': 'david', 'value': u'"new_value"', 'state': u'active'}) third_dictized['state'] = 'active' pprint(third_dictized) pprint(forth_dictized) assert third_dictized == forth_dictized
def _package_list_with_resources(context, package_revision_list): package_list = [] model = context["model"] for package in package_revision_list: result_dict = package_dictize(package,context) package_list.append(result_dict) return package_list
def package_show(context, data_dict): model = context['model'] context['session'] = model.Session name_or_id = data_dict.get("id") or data_dict['name_or_id'] pkg = model.Package.get(name_or_id) if pkg is None: raise NotFound context['package'] = pkg check_access('package_show', context, data_dict) package_dict = model_dictize.package_dictize(pkg, context) for item in plugins.PluginImplementations(plugins.IPackageController): item.read(pkg) schema = lib_plugins.lookup_package_plugin(package_dict['type']).db_to_form_schema() if schema: package_dict, errors = validate(package_dict, schema, context=context) return package_dict
def tag_show(context, data_dict): '''Shows tag details''' model = context['model'] api = context.get('api_version') or '1' id = data_dict['id'] tag = model.Tag.get(id) context['tag'] = tag if tag is None: raise NotFound check_access('tag_show',context, data_dict) tag_dict = tag_dictize(tag,context) extended_packages = [] for package in tag_dict['packages']: pkg = model.Package.get(package['id']) extended_packages.append(package_dictize(pkg,context)) tag_dict['packages'] = extended_packages return tag_dict
def _get_organizationpage_views(): organizationpage_view_ids = [] organizationpage_views = [] # list out all the resource ID whose is featured with package IDs in the organization try: resp = c.page.items if len(resp)>0 and resp[0].has_key('id'): for items in resp: pkg_id = items['id'] for view in db.Civicdata_Featured.find(organizationpage=True, package_id=pkg_id).all(): organizationpage_view_ids.append(view.resource_view_id) resource_views = model.Session.query(model.ResourceView).filter(model.ResourceView.id.in_(organizationpage_view_ids)).all() for view in resource_views: resource_view = md.resource_view_dictize(view, {'model': model}) resource_obj = model.Resource.get(resource_view['resource_id']) resource = md.resource_dictize(resource_obj, {'model': model}) organizationpage_views.append({ 'resource_view': resource_view, 'resource': resource, 'package': md.package_dictize(resource_obj.package, {'model':model}) }) except Exception, ex: print '\nDEBUG: '+str(ex)
def package_show(context, data_dict): model = context['model'] context['session'] = model.Session name_or_id = data_dict.get("id") or data_dict['name_or_id'] pkg = model.Package.get(name_or_id) if pkg is None: raise NotFound context['package'] = pkg check_access('package_show', context, data_dict) package_dict = model_dictize.package_dictize(pkg, context) for item in plugins.PluginImplementations(plugins.IPackageController): item.read(pkg) package_plugin = lib_plugins.lookup_package_plugin(package_dict['type']) try: schema = package_plugin.db_to_form_schema_options({ 'type':'show', 'api': 'api_version' in context, 'context': context }) except AttributeError: schema = package_plugin.db_to_form_schema() if schema and context.get('validate', True): package_dict, errors = validate(package_dict, schema, context=context) return package_dict
def test_10_package_alter_pending(self): context = {'model': model, 'session': model.Session, "user": '******', 'pending': True} anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed').one() anna_dictized = package_dictize(anna1, context) anna_dictized['name'] = u'annakarenina_changed2' anna_dictized['resources'][0]['url'] = u'http://new_url2' anna_dictized['tags'][0]['name'] = u'new_tag' anna_dictized['tags'][0].pop('id') #test if anna_dictized['extras'][0]['value'] = u'new_value' model.repo.new_revision() package_dict_save(anna_dictized, context) model.Session.commit() model.Session.remove() pkgrevisions = model.Session.query(model.PackageRevision).filter_by(id=anna1.id).all() sorted_packages = sorted(pkgrevisions, key=lambda x:x.revision_timestamp)[::-1] assert len(sorted_packages) == 3 assert sorted_packages[0].state == 'pending' assert sorted_packages[1].state == 'active' assert sorted_packages[1].current assert sorted_packages[2].state == 'active' assert str(sorted_packages[0].expired_timestamp) == '9999-12-31 00:00:00' assert str(sorted_packages[1].expired_timestamp) != '9999-12-31 00:00:00' assert str(sorted_packages[2].expired_timestamp) != '9999-12-31 00:00:00' resources_revisions = model.Session.query(model.ResourceRevision).filter_by(package_id=anna1.id).all() sorted_resources = sorted(resources_revisions, key=lambda x: (x.revision_timestamp, x.url))[::-1] for pkg in sorted_resources: print pkg.url, pkg.id, pkg.revision_timestamp, pkg.expired_timestamp, pkg.state, pkg.current assert len(sorted_resources) == 4 assert sorted_resources[0].state == 'pending' assert sorted_resources[1].state == 'active' assert sorted_resources[1].current assert sorted_resources[2].state == 'active' assert sorted_resources[2].current assert sorted_resources[3].state == 'active' assert str(sorted_resources[0].expired_timestamp) == '9999-12-31 00:00:00' assert str(sorted_resources[1].expired_timestamp) != '9999-12-31 00:00:00' assert str(sorted_resources[2].expired_timestamp) == '9999-12-31 00:00:00' assert str(sorted_resources[3].expired_timestamp) != '9999-12-31 00:00:00' tag_revisions = model.Session.query(model.PackageTagRevision).filter_by(package_id=anna1.id).all() sorted_tags = sorted(tag_revisions, key=lambda x: (x.revision_timestamp, x.tag.name))[::-1] print [(tag.state, tag.tag.name) for tag in sorted_tags]
def group_package_show(context, data_dict): """ Shows all packages belonging to a group. """ model = context["model"] user = context["user"] id = data_dict["id"] limit = data_dict.get("limit") group = model.Group.get(id) context["group"] = group if group is None: raise NotFound check_access("group_show", context, data_dict) query = ( model.Session.query(model.PackageRevision) .filter(model.PackageRevision.state == "active") .filter(model.PackageRevision.current == True) .join(model.PackageGroup, model.PackageGroup.package_id == model.PackageRevision.id) .join(model.Group, model.Group.id == model.PackageGroup.group_id) .filter_by(id=group.id) .order_by(model.PackageRevision.name) ) if limit: query = query.limit(limit) result = [] for pkg_rev in query.all(): result.append(package_dictize(pkg_rev, context)) return result
def group_package_show(context, data_dict): """ Shows all packages belonging to a group. """ model = context["model"] user = context["user"] id = data_dict['id'] limit = data_dict.get("limit") group = model.Group.get(id) context['group'] = group if group is None: raise NotFound check_access('group_show', context, data_dict) query = model.Session.query(model.PackageRevision)\ .filter(model.PackageRevision.state=='active')\ .filter(model.PackageRevision.current==True)\ .join(model.Member, model.Member.table_id==model.PackageRevision.id)\ .join(model.Group, model.Group.id==model.Member.group_id)\ .filter_by(id=group.id)\ .order_by(model.PackageRevision.name) if limit: query = query.limit(limit) if context.get('return_query'): return query result = [] for pkg_rev in query.all(): result.append(package_dictize(pkg_rev, context)) return result
def package_search(context, data_dict): model = context["model"] session = context["session"] user = context["user"] check_access("package_search", context, data_dict) # return a list of package ids data_dict["fl"] = "id" query = query_for(model.Package) query.run(data_dict) results = [] for package in query.results: # get the package object pkg_query = ( session.query(model.PackageRevision) .filter(model.PackageRevision.id == package) .filter(and_(model.PackageRevision.state == u"active", model.PackageRevision.current == True)) ) pkg = pkg_query.first() ## if the index has got a package that is not in ckan then ## ignore it. if not pkg: log.warning("package %s in index but not in database" % package) continue result_dict = package_dictize(pkg, context) results.append(result_dict) return {"count": query.count, "facets": query.facets, "results": results}
def test_package_dictize_resource(self): dataset = factories.Dataset() resource = factories.Resource(package_id=dataset['id'], name='test_pkg_dictize') dataset_obj = model.Package.get(dataset['id']) context = {'model': model, 'session': model.Session} result = model_dictize.package_dictize(dataset_obj, context) assert_equal_for_keys(result['resources'][0], resource, 'name', 'url') expected_dict = { u'cache_last_updated': None, u'cache_url': None, u'description': u'Just another test resource.', u'format': u'res_format', u'hash': u'', u'last_modified': None, u'mimetype': None, u'mimetype_inner': None, u'name': u'test_pkg_dictize', u'position': 0, u'resource_type': None, u'size': None, u'state': u'active', u'url': u'http://link.to.some.data', u'url_type': None, u'webstore_last_updated': None, u'webstore_url': None } self.assert_equals_expected(expected_dict, result['resources'][0])
def as_plain_dict(self, user, comment_count, updated, include_dataset=False, include_reports=False): '''Used for listing issues against a dataset Similar to as_dict, but we're not including full comments or the full user dict returns an issue_dict with a comment_count and a user as a string. ''' out = super(Issue, self).as_dict() # TODO: move this stuff to a schema try: out['abuse_status'] = AbuseStatus(out['abuse_status']).name except ValueError: pass out.update({ 'user': user, 'comment_count': comment_count, }) if isinstance(updated, datetime): out['updated'] = updated.isoformat() if include_dataset: pkg = self.dataset context = {'model': model, 'session': model.Session} out['dataset'] = model_dictize.package_dictize(pkg, context) if include_reports: out['abuse_reports'] = [i.user_id for i in self.abuse_reports] return out
def tag_show(context, data_dict): """Shows tag details""" model = context["model"] api = context.get("api_version") or "1" id = data_dict["id"] tag = model.Tag.get(id) context["tag"] = tag if tag is None: raise NotFound check_access("tag_show", context, data_dict) tag_dict = tag_dictize(tag, context) extended_packages = [] for package in tag_dict["packages"]: pkg = model.Package.get(package["id"]) extended_packages.append(package_dictize(pkg, context)) tag_dict["packages"] = extended_packages return tag_dict
def test_10_package_alter_pending(self): context = {"model": model, "session": model.Session, "pending": True} anna1 = model.Session.query(model.Package).filter_by(name="annakarenina_changed").one() anna_dictized = package_dictize(anna1, context) anna_dictized["name"] = u"annakarenina_changed2" anna_dictized["resources"][0]["url"] = u"new_url2" anna_dictized["tags"][0]["name"] = u"new_tag" anna_dictized["tags"][0].pop("id") # test if anna_dictized["extras"][0]["value"] = u'"new_value"' model.repo.new_revision() package_dict_save(anna_dictized, context) model.Session.commit() model.Session.remove() pkgrevisions = model.Session.query(model.PackageRevision).filter_by(id=anna1.id).all() sorted_packages = sorted(pkgrevisions, key=lambda x: x.revision_timestamp)[::-1] assert len(sorted_packages) == 3 assert sorted_packages[0].state == "pending" assert sorted_packages[1].state == "active" assert sorted_packages[1].current assert sorted_packages[2].state == "active" assert str(sorted_packages[0].expired_timestamp) == "9999-12-31 00:00:00" assert str(sorted_packages[1].expired_timestamp) != "9999-12-31 00:00:00" assert str(sorted_packages[2].expired_timestamp) != "9999-12-31 00:00:00" resources_revisions = ( model.Session.query(model.ResourceRevision).filter_by(resource_group_id=anna1.resource_groups[0].id).all() ) sorted_resources = sorted(resources_revisions, key=lambda x: (x.revision_timestamp, x.url))[::-1] for pkg in sorted_resources: print pkg.url, pkg.id, pkg.revision_timestamp, pkg.expired_timestamp, pkg.state, pkg.current assert len(sorted_resources) == 4 assert sorted_resources[0].state == "pending" assert sorted_resources[1].state == "active" assert sorted_resources[1].current assert sorted_resources[2].state == "active" assert sorted_resources[2].current assert sorted_resources[3].state == "active" assert str(sorted_resources[0].expired_timestamp) == "9999-12-31 00:00:00" assert str(sorted_resources[1].expired_timestamp) != "9999-12-31 00:00:00" assert str(sorted_resources[2].expired_timestamp) == "9999-12-31 00:00:00" assert str(sorted_resources[3].expired_timestamp) != "9999-12-31 00:00:00" tag_revisions = model.Session.query(model.PackageTagRevision).filter_by(package_id=anna1.id).all() sorted_tags = sorted(tag_revisions, key=lambda x: (x.revision_timestamp, x.tag.name))[::-1] print [(tag.state, tag.tag.name) for tag in sorted_tags]
def package_search(context, data_dict): model = context['model'] session = context['session'] user = context['user'] check_access('package_search', context, data_dict) # check if some extension needs to modify the search params for item in PluginImplementations(IPackageController): data_dict = item.before_search(data_dict) # the extension may have decided that it's no necessary to perform the query abort = data_dict.get('abort_search',False) results = [] if not abort: # return a list of package ids data_dict['fl'] = 'id' query = query_for(model.Package) query.run(data_dict) for package in query.results: # get the package object pkg_query = session.query(model.PackageRevision)\ .filter(model.PackageRevision.id == package)\ .filter(and_( model.PackageRevision.state == u'active', model.PackageRevision.current == True )) pkg = pkg_query.first() ## if the index has got a package that is not in ckan then ## ignore it. if not pkg: log.warning('package %s in index but not in database' % package) continue result_dict = package_dictize(pkg,context) results.append(result_dict) count = query.count facets = query.facets else: count = 0 facets = {} results = [] search_results = { 'count': count, 'facets': facets, 'results': results } # check if some extension needs to modify the search results for item in PluginImplementations(IPackageController): search_results = item.after_search(search_results,data_dict) return search_results
def test_package_openness_scores(self): context = {'model': model, 'session': model.Session} for p in model.Session.query(model.Package): context['id'] = p.id p = package_dictize(p, context) url = url_for('qa_dataset_action', action='five_stars') response = self.app.get(url) assert 'openness scores' in response, response
def test_package_dictize_title_stripped_of_whitespace(self): dataset = factories.Dataset(title=' has whitespace \t') dataset_obj = model.Package.get(dataset['id']) context = {'model': model, 'session': model.Session} result = model_dictize.package_dictize(dataset_obj, context) assert_equal(result['title'], 'has whitespace') assert_equal(dataset_obj.title, ' has whitespace \t')
def test_package_dictize_title_stripped_of_whitespace(self): dataset = factories.Dataset(title=" has whitespace \t") dataset_obj = model.Package.get(dataset["id"]) context = {"model": model, "session": model.Session} result = model_dictize.package_dictize(dataset_obj, context) assert result["title"] == "has whitespace" assert dataset_obj.title == " has whitespace \t"
def get_all_group_packages(self, group_id): """ Gets all of the group packages, public or private, returning them as a list of CKAN's dictized packages. """ result = [] for pkg_rev in model.Group.get(group_id).packages(with_private=True, context={'user_is_admin': True}): result.append(model_dictize.package_dictize(pkg_rev, {'model': model})) return result
def test_08_package_save(self): context = { "model": model, "user": "******", "session": model.Session, } anna1 = ( model.Session.query(model.Package) .filter_by(name="annakarenina") .one() ) anna_dictized = self.remove_changable_columns( package_dictize(anna1, context) ) anna_dictized["name"] = u"annakarenina3" package_dict_save(anna_dictized, context) model.Session.commit() # Re-clean anna_dictized anna_dictized = self.remove_changable_columns(anna_dictized) pkg = ( model.Session.query(model.Package) .filter_by(name="annakarenina3") .one() ) package_dictized = self.remove_changable_columns( package_dictize(pkg, context) ) anna_original = pformat(anna_dictized) anna_after_save = pformat(package_dictized) assert package_dictized == anna_dictized, "\n".join( unified_diff( anna_original.split("\n"), anna_after_save.split("\n") ) )
def test_09_package_alter(self): context = {"model": model, "session": model.Session} anna1 = model.Session.query( model.Package).filter_by(name='annakarenina').one() anna_dictized = package_dictize(anna1, context) anna_dictized["name"] = u'annakarenina_changed' anna_dictized["resources"][0]["url"] = u'new_url' model.repo.new_revision() package_dict_save(anna_dictized, context) model.Session.commit() model.Session.remove() pkg = model.Session.query( model.Package).filter_by(name='annakarenina_changed').one() package_dictized = package_dictize(pkg, context) resources_revisions = model.Session.query( model.ResourceRevision).filter_by( resource_group_id=anna1.resource_groups[0].id).all() sorted_resources = sorted(resources_revisions, key=lambda x: (x.revision_timestamp, x.url))[::-1] for res in sorted_resources: print res.id, res.revision_timestamp, res.expired_timestamp, res.state, res.current assert len(sorted_resources) == 3 anna_original = pformat(anna_dictized) anna_after_save = pformat(package_dictized) print anna_original print anna_after_save assert self.remove_changable_columns( anna_dictized) == self.remove_changable_columns(package_dictized) assert "\n".join( unified_diff(anna_original.split("\n"), anna_after_save.split("\n")))
def read(self, type_=None): context = {'model': model, 'session': model.Session, 'user': base.c.user or base.c.author} data_dict = {'id': base.c.user, 'user_obj': base.c.userobj} try: user_dict = logic.get_action('user_show')(context, data_dict) except NotFound: base.h.redirect_to(controller='user', action='login', id=None) except NotAuthorized: abort(401, _('Not authorized to see this page')) base.c.user_dict = user_dict base.c.is_myself = user_dict['name'] == base.c.user recommendation = r.Recommendation(base.c.userobj) recommendation.set_recommended_entity_class(str(prefix.void.Dataset.uri)) dataset_interests = set() subscription_query = model.Session.query(model.Subscription).filter(model.Subscription.owner_id==base.c.user_dict['id']) for subscription in subscription_query.all(): dataset_list = logic.get_action('subscription_dataset_list')(context, {'subscription_id': subscription.id}) for dataset in dataset_list: dataset_interests.add(h.dataset_to_uri(dataset['name'])) recommendation.set_additional_interests(dataset_interests) if type_ == None: types = ['topic', 'location', 'time'] recommendation.set_count_limit(2) else: types = [type_] recommendation.set_count_limit(10) base.c.recommendation = {} for type_ in types: recommendation.set_type(type_) recommendation.load() for dataset, reasons in recommendation.entities.iteritems(): dataset_dict = model_dictize.package_dictize(dataset, context) dataset_dict['dataset_reasons'] = [reason for reason in reasons if reason.class_uri == str(prefix.void.Dataset)] dataset_dict['dataset_reasons_count'] = len(dataset_dict['dataset_reasons']) dataset_dict['subscription_reasons'] = [reason for reason in reasons if reason.class_uri == str(prefix.ckan.Subscription)] dataset_dict['subscription_reasons_count'] = len(dataset_dict['subscription_reasons']) dataset_dict['multiple_reasons'] = dataset_dict['dataset_reasons_count'] + dataset_dict['subscription_reasons_count'] > 1 if base.c.recommendation.has_key(type_): base.c.recommendation[type_].append(dataset_dict) else: base.c.recommendation[type_] = [dataset_dict] return base.render('recommendation.html')
def package_search(context, data_dict): model = context['model'] session = context['session'] user = context['user'] check_access('package_search', context, data_dict) # check if some extension needs to modify the search params for item in plugins.PluginImplementations(plugins.IPackageController): data_dict = item.before_search(data_dict) # the extension may have decided that it's no necessary to perform the query abort = data_dict.get('abort_search', False) results = [] if not abort: # return a list of package ids data_dict['fl'] = 'id' query = search.query_for(model.Package) query.run(data_dict) for package in query.results: # get the package object pkg_query = session.query(model.PackageRevision)\ .filter(model.PackageRevision.id == package)\ .filter(and_( model.PackageRevision.state == u'active', model.PackageRevision.current == True )) pkg = pkg_query.first() ## if the index has got a package that is not in ckan then ## ignore it. if not pkg: log.warning('package %s in index but not in database' % package) continue result_dict = model_dictize.package_dictize(pkg, context) results.append(result_dict) count = query.count facets = query.facets else: count = 0 facets = {} results = [] search_results = {'count': count, 'facets': facets, 'results': results} # check if some extension needs to modify the search results for item in plugins.PluginImplementations(plugins.IPackageController): search_results = item.after_search(search_results, data_dict) return search_results
def get_all_group_packages(self, group_id, with_private=True): """ Gets all of the group packages, public or private, returning them as a list of CKAN's dictized packages. """ result = [] for pkg_rev in model.Group.get(group_id).packages(with_private=with_private, context={'user_is_admin': True}): result.append(model_dictize.package_dictize(pkg_rev, {'model': model})) return result
def generate_crawl(self): """ Generate strings using packages for crawling. Example: sudo /opt/data/ckan/pyenv/bin/paster --plugin=ckanext-kata katacmd crawl -c /etc/kata.ini "http://10.10.10.10:5000/dataset/%(name)s.rdf" """ if len(self.args) != 2: print "Crawl requires format argument" sys.exit(1) for package in model.Session.query(model.Package).all(): print self.args[1] % package_dictize(package, {'model': model})
def test_11_add_pending(self): context = {'model': model, 'session': model.Session, "user": '******', 'pending': True} anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed2').one() anna_dictized = package_dictize(anna1, context) anna_dictized['notes'] = 'wee' anna_dictized['resources'].append({ 'format': u'plain text', 'url': u'http://newurl'} ) anna_dictized['tags'].append({'name': u'newnew_tag'}) anna_dictized['extras'].append({'key': 'david', 'value': u'new_value'}) model.repo.new_revision() package_dict_save(anna_dictized, context) model.Session.commit() model.Session.remove() resources_revisions = model.Session.query(model.ResourceRevision).filter_by(package_id=anna1.id).all() sorted_resources = sorted(resources_revisions, key=lambda x: (x.revision_timestamp, x.url))[::-1] pprint(anna_dictized['resources']) for pkg in sorted_resources: print pkg.url, pkg.id, pkg.revision_timestamp, pkg.expired_timestamp, pkg.state, pkg.current assert len(sorted_resources) == 5, len(sorted_resources) assert sorted_resources[0].state == 'pending' assert sorted_resources[1].state == 'pending' assert sorted_resources[2].current assert sorted_resources[2].state == 'active' assert sorted_resources[3].current assert sorted_resources[3].state == 'active' assert sorted_resources[4].state == 'active' assert str(sorted_resources[0].expired_timestamp) == '9999-12-31 00:00:00' assert str(sorted_resources[1].expired_timestamp) == '9999-12-31 00:00:00' assert str(sorted_resources[2].expired_timestamp) != '9999-12-31 00:00:00' assert str(sorted_resources[3].expired_timestamp) == '9999-12-31 00:00:00' assert str(sorted_resources[4].expired_timestamp) != '9999-12-31 00:00:00' tag_revisions = model.Session.query(model.PackageTagRevision).filter_by(package_id=anna1.id).all() sorted_tags = sorted(tag_revisions, key=lambda x: (x.revision_timestamp, x.tag.name))[::-1] print [(tag.state, tag.tag.name) for tag in sorted_tags]
def test_package_dictize_license(self): dataset = factories.Dataset(license_id='cc-by') dataset_obj = model.Package.get(dataset['id']) context = {'model': model, 'session': model.Session} result = model_dictize.package_dictize(dataset_obj, context) assert_equal(result['isopen'], True) assert_equal(result['license_id'], 'cc-by') assert_equal(result['license_url'], 'http://www.opendefinition.org/licenses/cc-by') assert_equal(result['license_title'], 'Creative Commons Attribution')
def test_package_dictize_license(self): dataset = factories.Dataset(license_id="cc-by") dataset_obj = model.Package.get(dataset["id"]) context = {"model": model, "session": model.Session} result = model_dictize.package_dictize(dataset_obj, context) assert result["isopen"] assert result["license_id"] == "cc-by" assert (result["license_url"] == "http://www.opendefinition.org/licenses/cc-by") assert result["license_title"] == "Creative Commons Attribution"
def test_package_dictize_tags(self): dataset = factories.Dataset(tags=[{'name': 'fish'}]) dataset_obj = model.Package.get(dataset['id']) context = {'model': model, 'session': model.Session} result = model_dictize.package_dictize(dataset_obj, context) assert_equal(result['tags'][0]['name'], 'fish') expected_dict = {'display_name': u'fish', u'name': u'fish', u'state': u'active'} self.assert_equals_expected(expected_dict, result['tags'][0])
def get_package_for_resource(resource_id): context = {'model': base.model, 'session': base.model.Session} # get the matching resource object first resource_obj = base.model.Resource.get(resource_id) # get the package object that has the above resource related_pkg_obj = resource_obj.resource_group.package # convert the package object to dict package_dict = model_dictize.package_dictize(related_pkg_obj, context) return package_dict
def test_09_package_alter(self): context = {"model": model, "session": model.Session, "user": '******' } anna1 = model.Session.query(model.Package).filter_by(name='annakarenina').one() anna_dictized = package_dictize(anna1, context) anna_dictized["name"] = u'annakarenina_changed' anna_dictized["resources"][0]["url"] = u'http://new_url' model.repo.new_revision() package_dict_save(anna_dictized, context) model.Session.commit() model.Session.remove() pkg = model.Session.query(model.Package).filter_by(name='annakarenina_changed').one() package_dictized = package_dictize(pkg, context) resources_revisions = model.Session.query(model.ResourceRevision).filter_by(package_id=anna1.id).all() sorted_resources = sorted(resources_revisions, key=lambda x: (x.revision_timestamp, x.url))[::-1] for res in sorted_resources: print res.id, res.revision_timestamp, res.expired_timestamp, res.state, res.current assert len(sorted_resources) == 3 # Make sure we remove changeable fields BEFORE we store the pretty-printed version # for comparison clean_package_dictized = self.remove_changable_columns(package_dictized) anna_original = pformat(anna_dictized) anna_after_save = pformat(clean_package_dictized) assert self.remove_changable_columns(anna_dictized) == clean_package_dictized, \ "\n".join(unified_diff(anna_original.split("\n"), anna_after_save.split("\n")))
def test_09_package_alter(self): context = {"model": model, "session": model.Session, "user": "******"} anna1 = model.Session.query(model.Package).filter_by(name="annakarenina").one() anna_dictized = package_dictize(anna1, context) anna_dictized["name"] = u"annakarenina_changed" anna_dictized["resources"][0]["url"] = u"http://new_url" model.repo.new_revision() package_dict_save(anna_dictized, context) model.Session.commit() model.Session.remove() pkg = model.Session.query(model.Package).filter_by(name="annakarenina_changed").one() package_dictized = package_dictize(pkg, context) resources_revisions = ( model.Session.query(model.ResourceRevision) .filter_by(resource_group_id=anna1.resource_groups_all[0].id) .all() ) sorted_resources = sorted(resources_revisions, key=lambda x: (x.revision_timestamp, x.url))[::-1] for res in sorted_resources: print res.id, res.revision_timestamp, res.expired_timestamp, res.state, res.current assert len(sorted_resources) == 3 anna_original = pformat(anna_dictized) anna_after_save = pformat(package_dictized) print anna_original print anna_after_save assert self.remove_changable_columns(anna_dictized) == self.remove_changable_columns(package_dictized) assert "\n".join(unified_diff(anna_original.split("\n"), anna_after_save.split("\n")))
def test_02_package_dictize(self): context = {"model": model, "session": model.Session} model.Session.remove() pkg = model.Session.query(model.Package).filter_by(name="annakarenina").first() result = package_dictize(pkg, context) self.remove_changable_columns(result) print "\n".join(unified_diff(pformat(result).split("\n"), pformat(self.package_expected).split("\n"))) assert sorted(result.values()) == sorted(self.package_expected.values()) assert result == self.package_expected
def _prepare_entity_dict(self, _id, package=False): data_dict = {'id': _id} if package: package = model.Package.get(_id) package_dict = model_dictize.package_dictize(package, self.context) package_dict['num_followers'] = logic.get_action( 'dataset_follower_count')(self.context, data_dict) return package_dict group = model.Group.get(_id) group_dict = model_dictize.group_dictize(group, self.context) group_dict['num_followers'] = logic.get_action('group_follower_count')( self.context, data_dict) return group_dict
def test_02_package_dictize(self): context = {"model": model, "session": model.Session} model.Session.remove() pkg = model.Session.query(model.Package).filter_by(name='annakarenina').first() result = package_dictize(pkg, context) self.remove_changable_columns(result) print "\n".join(unified_diff(pformat(result).split("\n"), pformat(self.package_expected).split("\n"))) assert sorted(result.values()) == sorted(self.package_expected.values()) assert result == self.package_expected
def test_package_dictize_extras(self): extras_dict = {'key': 'latitude', 'value': '54.6'} dataset = factories.Dataset(extras=[extras_dict]) dataset_obj = model.Package.get(dataset['id']) context = {'model': model, 'session': model.Session} result = model_dictize.package_dictize(dataset_obj, context) assert_equal_for_keys(result['extras'][0], extras_dict, 'key', 'value') expected_dict = {u'key': u'latitude', u'state': u'active', u'value': u'54.6'} self.assert_equals_expected(expected_dict, result['extras'][0])
def test_package_dictize_tags(self): dataset = factories.Dataset(tags=[{"name": "fish"}]) dataset_obj = model.Package.get(dataset["id"]) context = {"model": model, "session": model.Session} result = model_dictize.package_dictize(dataset_obj, context) assert result["tags"][0]["name"] == "fish" expected_dict = { "display_name": u"fish", u"name": u"fish", u"state": u"active", } self.assert_equals_expected(expected_dict, result["tags"][0])
def test_11_add_pending(self): context = {"model": model, "session": model.Session, "user": "******", "pending": True} anna1 = model.Session.query(model.Package).filter_by(name="annakarenina_changed2").one() anna_dictized = package_dictize(anna1, context) anna_dictized["notes"] = "wee" anna_dictized["resources"].append({"format": u"plain text", "url": u"http://newurl"}) anna_dictized["tags"].append({"name": u"newnew_tag"}) anna_dictized["extras"].append({"key": "david", "value": u"new_value"}) model.repo.new_revision() package_dict_save(anna_dictized, context) model.Session.commit() model.Session.remove() resources_revisions = ( model.Session.query(model.ResourceRevision) .filter_by(resource_group_id=anna1.resource_groups_all[0].id) .all() ) sorted_resources = sorted(resources_revisions, key=lambda x: (x.revision_timestamp, x.url))[::-1] pprint(anna_dictized["resources"]) for pkg in sorted_resources: print pkg.url, pkg.id, pkg.revision_timestamp, pkg.expired_timestamp, pkg.state, pkg.current assert len(sorted_resources) == 5, len(sorted_resources) assert sorted_resources[0].state == "pending" assert sorted_resources[1].state == "pending" assert sorted_resources[2].current assert sorted_resources[2].state == "active" assert sorted_resources[3].current assert sorted_resources[3].state == "active" assert sorted_resources[4].state == "active" assert str(sorted_resources[0].expired_timestamp) == "9999-12-31 00:00:00" assert str(sorted_resources[1].expired_timestamp) == "9999-12-31 00:00:00" assert str(sorted_resources[2].expired_timestamp) != "9999-12-31 00:00:00" assert str(sorted_resources[3].expired_timestamp) == "9999-12-31 00:00:00" assert str(sorted_resources[4].expired_timestamp) != "9999-12-31 00:00:00" tag_revisions = model.Session.query(model.PackageTagRevision).filter_by(package_id=anna1.id).all() sorted_tags = sorted(tag_revisions, key=lambda x: (x.revision_timestamp, x.tag.name))[::-1] print [(tag.state, tag.tag.name) for tag in sorted_tags]
def test_26_package_dictize_whitespace_strippped_from_title(self): context = {"model": model, "session": model.Session} pkg = model.Session.query(model.Package).first() original_title = pkg.title pkg.title = " whitespace title \t" model.Session.add(pkg) model.Session.commit() result = package_dictize(pkg, context) assert result['title'] == 'whitespace title' pkg.title = original_title model.Session.add(pkg) model.Session.commit()
def test_package_dictize_extras(self): extras_dict = {"key": "latitude", "value": "54.6"} dataset = factories.Dataset(extras=[extras_dict]) dataset_obj = model.Package.get(dataset["id"]) context = {"model": model, "session": model.Session} result = model_dictize.package_dictize(dataset_obj, context) assert_equal_for_keys(result["extras"][0], extras_dict, "key", "value") expected_dict = { u"key": u"latitude", u"state": u"active", u"value": u"54.6", } self.assert_equals_expected(expected_dict, result["extras"][0])
def showcase_list(context, data_dict): '''Return a list of all showcases in the site.''' toolkit.check_access('ckanext_showcase_list', context, data_dict) model = context["model"] q = model.Session.query(model.Package) \ .filter(model.Package.type == 'showcase') \ .filter(model.Package.state == 'active') showcase_list = [] for pkg in q.all(): showcase_list.append(model_dictize.package_dictize(pkg, context)) return showcase_list
def package_create(context, data_dict): model = context['model'] user = context['user'] preview = context.get('preview', False) schema = context.get('schema') or default_create_package_schema() model.Session.remove() model.Session()._context = context check_access(model.System(), model.Action.PACKAGE_CREATE, context) check_group_auth(context, data_dict) data, errors = validate(data_dict, schema, context) if errors: model.Session.rollback() raise ValidationError(errors, package_error_summary(errors)) if not preview: rev = model.repo.new_revision() rev.author = user if 'message' in context: rev.message = context['message'] else: rev.message = _(u'REST API: Create object %s') % data.get("name") pkg = package_dict_save(data, context) admins = [] if user: admins = [model.User.by_name(user.decode('utf8'))] if not preview: model.setup_default_user_roles(pkg, admins) for item in PluginImplementations(IPackageController): item.create(pkg) model.repo.commit() ## need to let rest api create and preview context["package"] = pkg ## this is added so that the rest controller can make a new location context["id"] = pkg.id log.debug('Created object %s' % str(pkg.name)) if not preview: return package_dictize(pkg, context) else: return data
def test_package_dictize_basic(self): dataset = factories.Dataset( name="test_dataset_dictize", notes="Some *description*", url="http://example.com", ) dataset_obj = model.Package.get(dataset["id"]) context = {"model": model, "session": model.Session} result = model_dictize.package_dictize(dataset_obj, context) assert result["name"] == dataset["name"] assert not (result["isopen"]) assert result["type"] == dataset["type"] today = datetime.date.today().strftime("%Y-%m-%d") assert result["metadata_modified"].startswith(today) assert result["metadata_created"].startswith(today) assert result["creator_user_id"] == dataset_obj.creator_user_id expected_dict = { "author": None, "author_email": None, "extras": [], "groups": [], "isopen": False, "license_id": None, "license_title": None, "maintainer": None, "maintainer_email": None, "name": u"test_dataset_dictize", "notes": "Some *description*", "num_resources": 0, "num_tags": 0, "organization": None, "owner_org": None, "private": False, "relationships_as_object": [], "relationships_as_subject": [], "resources": [], "state": u"active", "tags": [], "title": u"Test Dataset", "type": u"dataset", "url": "http://example.com", "version": None, } self.assert_equals_expected(expected_dict, result)
def apiset_list(context, data_dict): model = context["model"] q = ( model.Session.query( model.Package).filter(model.Package.type == 'apiset').filter( model.Package.private == False) # noqa .filter(model.Package.state == 'active')) limit = data_dict.get('limit') if limit: q = q.limit(limit) offset = data_dict.get('offset') if offset: q = q.offset(offset) return [model_dictize.package_dictize(pkg, context) for pkg in q.all()]
def test_package_dictize_basic(self): dataset = factories.Dataset(name='test_dataset_dictize', notes='Some *description*', url='http://example.com') dataset_obj = model.Package.get(dataset['id']) context = {'model': model, 'session': model.Session} result = model_dictize.package_dictize(dataset_obj, context) assert_equal(result['name'], dataset['name']) assert_equal(result['isopen'], False) assert_equal(result['type'], dataset['type']) today = datetime.date.today().strftime('%Y-%m-%d') assert result['metadata_modified'].startswith(today) assert result['metadata_created'].startswith(today) assert_equal(result['creator_user_id'], dataset_obj.creator_user_id) expected_dict = { 'author': None, 'author_email': None, 'extras': [], 'groups': [], 'isopen': False, 'license_id': None, 'license_title': None, 'maintainer': None, 'maintainer_email': None, 'name': u'test_dataset_dictize', 'notes': 'Some *description*', 'num_resources': 0, 'num_tags': 0, 'organization': None, 'owner_org': None, 'private': False, 'relationships_as_object': [], 'relationships_as_subject': [], 'resources': [], 'state': u'active', 'tags': [], 'title': u'Test Dataset', 'type': u'dataset', 'url': 'http://example.com', 'version': None } self.assert_equals_expected(expected_dict, result)
def get_deleted_datasets_filtered(self): # TODO: this was the code to retrive the deleted datasets # Query result is a list of tuples, each tuple contains only the dataset id as elements deleted_datasets = ckan.model.Session.query(Package). \ join(Group, Group.id == Package.owner_org). \ filter(Package.state == 'deleted'). \ join(Member, Member.group_id == Group.id). \ filter(Member.table_name == 'user'). \ join(User, User.id == Member.table_id). \ filter(User.name == c.user).all() # Adapt deleted datasets representation anf filter if needed adapted_deleted_datasets = [] for ds in deleted_datasets: ds_dict = model_dictize.package_dictize(ds, {'model': ckan.model}) # Add missing properties to circumvent access errors / emtpy fields in thr Jinja templates if not ds_dict.get('tracking_summary', None): ds_dict['tracking_summary'] = {'total': 0} if not ds_dict.get('description', None): ds_dict['description'] = ds.notes adapted_deleted_datasets.append(ds_dict) # Only keep datasets matching the criteria adapted_filtered_deleted_datasets = [] for ds in adapted_deleted_datasets: ds_title = ds.get('title', '') if c.q != u'': search = c.q.strip().split(' ') if request.params.get('ext_boolean') == 'any': if any([token in ds_title for token in search]): adapted_filtered_deleted_datasets.append(ds) elif request.params.get('ext_boolean') == 'all': if all([token in ds_title for token in search]): adapted_filtered_deleted_datasets.append(ds) elif request.params.get('ext_boolean') == 'exact': if " ".join(search) == ds_title: adapted_filtered_deleted_datasets.append(ds) else: adapted_filtered_deleted_datasets.append(ds) log.debug("MDT: Deleted datasets adapted and filtered") return adapted_filtered_deleted_datasets
def package_update(context, data_dict): model = context['model'] user = context['user'] id = data_dict["id"] preview = context.get('preview', False) schema = context.get('schema') or default_update_package_schema() model.Session.remove() model.Session()._context = context pkg = model.Package.get(id) context["package"] = pkg if pkg is None: raise NotFound(_('Package was not found.')) data_dict["id"] = pkg.id check_access(pkg, model.Action.EDIT, context) data, errors = validate(data_dict, schema, context) check_group_auth(context, data) if errors: model.Session.rollback() raise ValidationError(errors, package_error_summary(errors)) if not preview: rev = model.repo.new_revision() rev.author = user if 'message' in context: rev.message = context['message'] else: rev.message = _(u'REST API: Update object %s') % data.get("name") pkg = package_dict_save(data, context) if not preview: for item in PluginImplementations(IPackageController): item.edit(pkg) model.repo.commit() return package_dictize(pkg, context) return data
def test_package_dictization_with_deleted_group(self): """ Ensure that the dictization does not return groups that the dataset has been removed from. """ pkg = model.Package(name="testing-deleted-groups") group_1 = model.Group(name="test-group-1") group_2 = model.Group(name="test-group-2") model.Session.add(pkg) model.Session.add(group_1) model.Session.add(group_2) model.Session.flush() # Add the dataset to group_1, and signal that the dataset used # to be a member of group_2 by setting its membership state to 'deleted' membership_1 = model.Member( table_id=pkg.id, table_name="package", group=group_1, group_id=group_1.id, state="active", ) membership_2 = model.Member( table_id=pkg.id, table_name="package", group=group_2, group_id=group_2.id, state="deleted", ) model.Session.add(membership_1) model.Session.add(membership_2) model.repo.commit() # Dictize the dataset context = {"model": model, "session": model.Session} result = package_dictize(pkg, context) self.remove_changable_columns(result) assert "test-group-2" not in [g["name"] for g in result["groups"]] assert "test-group-1" in [g["name"] for g in result["groups"]]
def package_show(context, data_dict): model = context['model'] api = context.get('api_version') or '1' id = data_dict['id'] pkg = model.Package.get(id) context['package'] = pkg if pkg is None: raise NotFound check_access(pkg, model.Action.READ, context) package_dict = package_dictize(pkg, context) for item in PluginImplementations(IPackageController): item.read(pkg) return package_dict
def import_stage(self, harvest_object): import_stage = super(MultilangHarvester, self).import_stage(harvest_object) if import_stage == True: # Get the existing HarvestObject existing_object = model.Session.query(HarvestObject) \ .filter(HarvestObject.guid==harvest_object.guid) \ .filter(HarvestObject.current==True) \ .first() session = Session harvested_package = session.query(Package).filter(Package.id == existing_object.package_id).first() if harvested_package: package_dict = model_dictize.package_dictize(harvested_package, {'model': model, 'session': session}) if package_dict: self.after_import_stage(package_dict) return import_stage
def package_show(context, data_dict): model = context['model'] name_or_id = data_dict.get("id") or data_dict['name_or_id'] pkg = model.Package.get(name_or_id) if pkg is None: raise NotFound context['package'] = pkg check_access('package_show', context, data_dict) package_dict = model_dictize.package_dictize(pkg, context) for item in plugins.PluginImplementations(plugins.IPackageController): item.read(pkg) return package_dict
def showcase_list(context, data_dict): '''Return a list of all showcases in the site.''' toolkit.check_access('ckanext_showcase_list', context, data_dict) model = context["model"] q = model.Session.query(model.Package) \ .filter(model.Package.type == 'showcase') \ .filter(model.Package.state == 'active') # Showcase includes private showcases by default, but those can be excluded with include_private = false if data_dict.get('include_private') == 'false': q = q.filter(model.Package.private == False) showcase_list = [] for pkg in q.all(): showcase_list.append(model_dictize.package_dictize(pkg, context)) return showcase_list
def test_21_package_dictization_with_deleted_group(self): """ Ensure that the dictization does not return groups that the dataset has been removed from. """ # Create a new dataset and 2 new groups model.repo.new_revision() pkg = model.Package(name='testing-deleted-groups') group_1 = model.Group(name='test-group-1') group_2 = model.Group(name='test-group-2') model.Session.add(pkg) model.Session.add(group_1) model.Session.add(group_2) model.Session.flush() # Add the dataset to group_1, and signal that the dataset used # to be a member of group_2 by setting its membership state to 'deleted' membership_1 = model.Member(table_id = pkg.id, table_name = 'package', group = group_1, group_id = group_1.id, state = 'active') membership_2 = model.Member(table_id = pkg.id, table_name = 'package', group = group_2, group_id = group_2.id, state = 'deleted') model.Session.add(membership_1) model.Session.add(membership_2) model.repo.commit() # Dictize the dataset context = {"model": model, "session": model.Session} result = package_dictize(pkg, context) self.remove_changable_columns(result) assert_not_in('test-group-2', [ g['name'] for g in result['groups'] ]) assert_in('test-group-1', [ g['name'] for g in result['groups'] ])