def test_2_group_schema(self): group = model.Session.query(model.Group).first() data = group_dictize(group, self.context) converted_data, errors = validate(data, default_group_schema(), self.context) group_pack = sorted(group.packages, key=lambda x:x.id) converted_data["packages"] = sorted(converted_data["packages"], key=lambda x:x["id"]) expected = {'description': u'These are books that David likes.', 'id': group.id, 'name': u'david', 'packages': sorted([{'id': group_pack[0].id}, {'id': group_pack[1].id, }], key=lambda x:x["id"]), 'title': u"Dave's books"} assert not errors assert converted_data == expected, pformat(converted_data) + '\n\n' + pformat(expected) data["packages"].sort(key=lambda x:x["id"]) data["packages"][0]["id"] = 'fjdlksajfalsf' data["packages"][1].pop("id") data["packages"][1].pop("name") converted_data, errors = validate(data, default_group_schema(), self.context) assert errors == {'packages': [{'id': [u'Dataset was not found.']}, {'id': [u'Missing value']}]} , pformat(errors)
def test_simple(self): raise SkipTest() schema = {"name": [not_empty], "age": [ignore_missing, convert_int], "gender": [default("female")]} data = {"name": "fred", "age": "32"} converted_data, errors = validate(data, schema) assert not errors assert converted_data == {"gender": "female", "age": 32, "name": "fred"}, converted_data data = {"name": "", "age": "dsa32", "extra": "extra"} converted_data, errors = validate(data, schema) assert errors == {("age",): [u"Please enter an integer value"], ("name",): [u"Missing value"]}, errors assert converted_data == {"gender": "female", "age": "dsa32", "name": "", "__extras": {"extra": "extra"}} data = {"name": "fred", "numbers": [{"number": "13221312"}, {"number": "432423432", "code": "+44"}]} schema = {"name": [not_empty], "numbers": {"number": [convert_int], "code": [not_empty], "__extras": [ignore]}} converted_data, errors = validate(data, schema) assert errors == {("numbers", 0, "code"): ["Missing value"]}
def testUserBothPasswordsEnteredForOrderIndependence(self): #Verify that validator works, even if it's declared on one, the other or both password fields data = { 'password1':'', 'password2':'notempty' } schema = { 'password1':[v.user_both_passwords_entered] } converted_data, errors = df.validate(data, schema) self.assertEqual(errors, {'password1':[u'Please enter both passwords']}) schema = { 'password2':[v.user_both_passwords_entered] } converted_data, errors = df.validate(data, schema) self.assertEqual(errors, {'password2':[u'Please enter both passwords']}) schema = { 'password2':[v.user_both_passwords_entered], 'password1':[v.user_both_passwords_entered] } converted_data, errors = df.validate(data, schema) self.assertEqual(errors, {'password1':[u'Please enter both passwords'], 'password2':[u'Please enter both passwords']}) self.assertEqual(data, converted_data)
def test_tag_string_parsing(self): # 'tag_string' is what you type into the tags field in the package # edit form. This test checks that it is parsed correctly or reports # errors correctly. context = {'model': ckan.model, 'session': ckan.model.Session} schema = ckan.logic.schema.default_update_package_schema() # basic parsing of comma separated values tests = (('tag', ['tag'], []), ('tag1, tag2', ['tag1', 'tag2'], []), ('tag 1', ['tag 1'], []), ) for tag_string, expected_tags, expected_errors in tests: data_dict = {'tag_string': tag_string} data, errors = validate(data_dict, schema, context) assert_equal(errors.get('tags', []), expected_errors) tag_names = [tag_dict['name'] for tag_dict in data['tags']] assert_equal(tag_names, expected_tags) # test whitespace chars are stripped whitespace_characters = u'\t\n\r\f\v ' for ch in whitespace_characters: tag = ch + u'tag name' + ch data_dict = {'tag_string': tag} data, errors = validate(data_dict, schema, context) assert_equal(data['tags'], [{'name': u'tag name'}])
def test_2_group_schema(self): context = {"model": model, "session": model.Session} group = model.Session.query(model.Group).first() data = group_dictize(group, context) converted_data, errors = validate(data, default_group_schema(), context) group_pack = sorted(group.packages, key=lambda x: x.id) converted_data["packages"] = sorted(converted_data["packages"], key=lambda x: x["id"]) expected = { "description": u"These are books that David likes.", "id": group.id, "name": u"david", "packages": sorted([{"id": group_pack[0].id}, {"id": group_pack[1].id}], key=lambda x: x["id"]), "title": u"Dave's books", } assert not errors assert converted_data == expected, pformat(converted_data) + "\n\n" + pformat(expected) data["packages"].sort(key=lambda x: x["id"]) data["packages"][0]["id"] = "fjdlksajfalsf" data["packages"][1].pop("id") data["packages"][1].pop("name") converted_data, errors = validate(data, default_group_schema(), context) assert errors == {"packages": [{"id": [u"Dataset was not found."]}, {"id": [u"Missing value"]}]}, pformat( errors )
def testFieldValidationPublicAccessLevelPublic(self): data = {'public_access_level':'public' } schema = self.__getSchemaFromMetadataDict__('public_access_level') converted_data, errors = df.validate(data, schema) self.assertEqual(errors, {}) data['public_access_level']='Public' converted_data, errors = df.validate(data, schema) self.assertEqual(errors, {})
def test_2_group_schema(self): group = model.Session.query(model.Group).first() data = group_dictize(group, self.context) # we don't want these here del data['groups'] del data['users'] del data['tags'] del data['extras'] converted_data, errors = validate(data, default_group_schema(), self.context) group_pack = sorted(group.packages(), key=lambda x: x.id) converted_data["packages"] = sorted(converted_data["packages"], key=lambda x: x["id"]) expected = { 'description': u'These are books that David likes.', 'id': group.id, 'name': u'david', 'is_organization': False, 'type': u'group', 'image_url': u'', 'image_display_url': u'', 'packages': sorted([{'id': group_pack[0].id, 'name': group_pack[0].name, 'title': group_pack[0].title}, {'id': group_pack[1].id, 'name': group_pack[1].name, 'title':group_pack[1].title}], key=lambda x: x["id"]), 'title': u"Dave's books", 'approval_status': u'approved' } assert not errors assert converted_data == expected, pformat(converted_data) data["packages"].sort(key=lambda x: x["id"]) data["packages"][0]["id"] = 'fjdlksajfalsf' data["packages"][1].pop("id") data["packages"][1].pop("name") converted_data, errors = validate(data, default_group_schema(), self.context) assert errors == { 'packages': [{'id': [u'Not found: Dataset']}, {'id': [u'Missing value']}] }, pformat(errors)
def test_simple(): schema = { "name": [not_empty], "age": [ignore_missing, convert_int], "gender": [default("female")], } data = { "name": "fred", "age": "32", } converted_data, errors = validate(data, schema) assert not errors assert converted_data == {'gender': 'female', 'age': 32, 'name': 'fred'}, converted_data data = { "name": "", "age": "dsa32", "extra": "extra", } converted_data, errors = validate(data, schema) assert errors == {'age': [u'Please enter an integer value'], 'name': [u'Missing value']}, errors assert converted_data == {'gender': 'female', 'age': 'dsa32', 'name': '', '__extras': {'extra': 'extra'}} data = {"name": "fred", "numbers": [{"number": "13221312"}, {"number": "432423432", "code": "+44"}] } schema = { "name": [not_empty], "numbers": { "number": [convert_int], "code": [not_empty], "__extras": [ignore], } } converted_data, errors = validate(data, schema) print(errors) assert errors == {'numbers': [{'code': [u'Missing value']}, {}]}
def test_4_tag_schema_allows_limited_punctuation(self): """Asserts that a tag name with limited punctuation is valid""" ignored = "" data = {"name": u".-_", "revision_timestamp": ignored, "state": ignored} _, errors = validate(data, default_tags_schema(), self.context) assert not errors, str(errors)
def edit(self, id, data=None, errors=None, error_summary=None): package_type = self._get_package_type(id) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'extras_as_string': True, 'save': 'save' in request.params, 'moderated': config.get('moderated'), 'pending': True,} if context['save'] and not data: return self._save_edit(id, context) try: old_data = get_action('package_show')(context, {'id':id}) schema = self._db_to_form_schema(package_type=package_type) if schema and not data: old_data, errors = validate(old_data, schema, context=context) data = data or old_data # Merge all elements for the complete package dictionary c.pkg_dict = dict(old_data.items() + data.items()) except NotAuthorized: abort(401, _('Unauthorized to read package %s') % '') except NotFound: abort(404, _('Dataset not found')) c.pkg = context.get("package") c.pkg_json = json.dumps(data) try: check_access('package_update',context) except NotAuthorized, e: abort(401, _('User %r not authorized to edit %s') % (c.user, id))
def edit(self, id, data=None, errors=None, error_summary=None): context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'extras_as_string': True, 'save': 'save' in request.params, 'schema': self._form_to_db_schema(), } data_dict = {'id': id} if context['save'] and not data: return self._save_edit(id, context) try: old_data = get_action('group_show')(context, data_dict) c.grouptitle = old_data.get('title') c.groupname = old_data.get('name') schema = self._db_to_form_schema() if schema and not data: old_data, errors = validate(old_data, schema, context=context) data = data or old_data except NotFound: abort(404, _('Group not found')) except NotAuthorized: abort(401, _('Unauthorized to read group %s') % '') group = context.get("group") c.group = group try: check_access('group_update',context) except NotAuthorized, e: abort(401, _('User %r not authorized to edit %s') % (c.user, id))
def user_update(context, data_dict): '''Updates the user\'s details''' model = context['model'] user = context['user'] schema = context.get('schema') or default_update_user_schema() id = data_dict['id'] user_obj = model.User.get(id) context['user_obj'] = user_obj if user_obj is None: raise NotFound('User was not found.') check_access('user_update', context, data_dict) data, errors = validate(data_dict, schema, context) if errors: model.Session.rollback() raise ValidationError(errors, group_error_summary(errors)) user = user_dict_save(data, context) if not context.get('defer_commit'): model.repo.commit() return user_dictize(user, context)
def task_status_update(context, data_dict): model = context['model'] session = model.meta.create_local_session() context['session'] = session user = context['user'] id = data_dict.get("id") schema = context.get('schema') or default_task_status_schema() if id: task_status = model.TaskStatus.get(id) context["task_status"] = task_status if task_status is None: raise NotFound(_('TaskStatus was not found.')) check_access('task_status_update', context, data_dict) data, errors = validate(data_dict, schema, context) if errors: session.rollback() raise ValidationError(errors, task_status_error_summary(errors)) task_status = task_status_dict_save(data, context) session.commit() session.close() return task_status_dictize(task_status, context)
def resource_update(context, data_dict): model = context['model'] session = context['session'] user = context['user'] id = data_dict["id"] schema = context.get('schema') or default_update_resource_schema() model.Session.remove() resource = model.Resource.get(id) context["resource"] = resource if not resource: raise NotFound(_('Resource was not found.')) check_access('resource_update', context, data_dict) data, errors = validate(data_dict, schema, context) if errors: model.Session.rollback() raise ValidationError(errors, resource_error_summary(errors)) rev = model.repo.new_revision() rev.author = user if 'message' in context: rev.message = context['message'] else: rev.message = _(u'REST API: Update object %s') % data.get("name") resource = resource_dict_save(data, context) if not context.get('defer_commit'): model.repo.commit() return resource_dictize(resource, context)
def package_update_validate(context, data_dict): model = context['model'] user = context['user'] id = data_dict["id"] schema = context.get('schema') or default_update_package_schema() model.Session.remove() model.Session()._context = context pkg = model.Package.get(id) context["package"] = pkg if pkg is None: raise NotFound(_('Package was not found.')) data_dict["id"] = pkg.id check_access('package_update', context, data_dict) data, errors = validate(data_dict, schema, context) if errors: model.Session.rollback() raise ValidationError(errors, package_error_summary(errors)) return data
def term_translation_update(context, data_dict): model = context["model"] check_access("term_translation_update", context, data_dict) schema = { "term": [validators.not_empty, unicode], "term_translation": [validators.not_empty, unicode], "lang_code": [validators.not_empty, unicode], } data, errors = validate(data_dict, schema, context) if errors: model.Session.rollback() raise ValidationError(errors) trans_table = model.term_translation_table update = trans_table.update() update = update.where(trans_table.c.term == data["term"]) update = update.where(trans_table.c.lang_code == data["lang_code"]) update = update.values(term_translation=data["term_translation"]) conn = model.Session.connection() result = conn.execute(update) # insert if not updated if not result.rowcount: conn.execute(trans_table.insert().values(**data)) if not context.get("defer_commit"): model.Session.commit() return data
def vocabulary_update(context, data_dict): model = context["model"] vocab_id = data_dict.get("id") if not vocab_id: raise ValidationError({"id": _("id not in data")}) vocab = model.vocabulary.Vocabulary.get(vocab_id) if vocab is None: raise NotFound(_('Could not find vocabulary "%s"') % vocab_id) data_dict["id"] = vocab.id if data_dict.has_key("name"): if data_dict["name"] == vocab.name: del data_dict["name"] check_access("vocabulary_update", context, data_dict) schema = context.get("schema") or default_update_vocabulary_schema() data, errors = validate(data_dict, schema, context) if errors: model.Session.rollback() raise ValidationError(errors) updated_vocab = vocabulary_dict_update(data, context) if not context.get("defer_commit"): model.repo.commit() return vocabulary_dictize(updated_vocab, context)
def package_relationship_update(context, data_dict): model = context["model"] user = context["user"] schema = context.get("schema") or default_update_relationship_schema() api = context.get("api_version") or "1" id = data_dict["subject"] id2 = data_dict["object"] rel = data_dict["type"] ref_package_by = "id" if api == "2" else "name" pkg1 = model.Package.get(id) pkg2 = model.Package.get(id2) if not pkg1: raise NotFound("Subject package %r was not found." % id) if not pkg2: return NotFound("Object package %r was not found." % id2) data, errors = validate(data_dict, schema, context) if errors: model.Session.rollback() raise ValidationError(errors, relationship_error_summary(errors)) check_access("package_relationship_update", context, data_dict) existing_rels = pkg1.get_relationships_with(pkg2, rel) if not existing_rels: raise NotFound("This relationship between the packages was not found.") entity = existing_rels[0] comment = data_dict.get("comment", u"") context["relationship"] = entity return _update_package_relationship(entity, comment, context)
def user_update(context, data_dict): """Updates the user\'s details""" model = context["model"] user = context["user"] session = context["session"] schema = context.get("schema") or default_update_user_schema() id = data_dict["id"] user_obj = model.User.get(id) context["user_obj"] = user_obj if user_obj is None: raise NotFound("User was not found.") check_access("user_update", context, data_dict) data, errors = validate(data_dict, schema, context) if errors: session.rollback() raise ValidationError(errors, group_error_summary(errors)) user = user_dict_save(data, context) activity_dict = {"user_id": user.id, "object_id": user.id, "activity_type": "changed user"} activity_create_context = {"model": model, "user": user, "defer_commit": True, "session": session} from ckan.logic.action.create import activity_create activity_create(activity_create_context, activity_dict, ignore_auth=True) # TODO: Also create an activity detail recording what exactly changed in # the user. if not context.get("defer_commit"): model.repo.commit() return user_dictize(user, context)
def test_convert_to_extras_field_can_be_combined_with_extras_deleted(self): data_dict = { 'custom_text': 'Hi', 'extras': [ {'key': 'proper_extra', 'value': 'Bye', 'deleted': True}, {'key': 'proper_extra2', 'value': 'Bye2'}, ] } schema = { 'custom_text': [convert_to_extras], 'extras': default_extras_schema(), } context = { 'model': model, 'session': model.Session, } data, errors = validate(data_dict, schema, context) assert 'extras' in data eq_(len(data['extras']), 3) eq_(sorted([e['key'] for e in data['extras']]), ['custom_text', 'proper_extra', 'proper_extra2']) eq_(sorted([e['value'] for e in data['extras']]), ['Bye', 'Bye2', 'Hi'])
def package_showcase_list(context, data_dict): '''List showcases associated with a package. :param package_id: id or name of the package :type package_id: string :rtype: list of dictionaries ''' toolkit.check_access('ckanext_package_showcase_list', context, data_dict) # validate the incoming data_dict validated_data_dict, errors = validate(data_dict, package_showcase_list_schema(), context) if errors: raise toolkit.ValidationError(errors) # get a list of showcase ids associated with the package id showcase_id_list = ShowcasePackageAssociation.get_showcase_ids_for_package( validated_data_dict['package_id']) showcase_list = [] if showcase_id_list is not None: for showcase_id in showcase_id_list: try: showcase = toolkit.get_action('package_show')(context, {'id': showcase_id}) showcase_list.append(showcase) except NotAuthorized: pass return showcase_list
def harvest_source_extra_validator(key,data,errors,context): harvester_type = data.get(('source_type',),'') #gather all extra fields to use as whitelist of what #can be added to top level data_dict all_extra_fields = set() for harvester in PluginImplementations(IHarvester): if not hasattr(harvester, 'extra_schema'): continue all_extra_fields.update(harvester.extra_schema().keys()) extra_schema = {'__extras': [keep_not_empty_extras]} for harvester in PluginImplementations(IHarvester): if not hasattr(harvester, 'extra_schema'): continue info = harvester.info() if not info['name'] == harvester_type: continue extra_schema.update(harvester.extra_schema()) break extra_data, extra_errors = validate(data.get(key, {}), extra_schema) for key in extra_data.keys(): #only allow keys that appear in at least one harvester if key not in all_extra_fields: extra_data.pop(key) for key, value in extra_data.iteritems(): data[(key,)] = value for key, value in extra_errors.iteritems(): errors[(key,)] = value
def project_package_list(context, data_dict): '''List packages associated with a project. :param project_id: id or name of the project :type project_id: string :rtype: list of dictionaries ''' toolkit.check_access('ckanext_project_package_list', context, data_dict) # validate the incoming data_dict validated_data_dict, errors = validate(data_dict, project_package_list_schema(), context) if errors: raise toolkit.ValidationError(errors) # get a list of package ids associated with project id pkg_id_list = projectPackageAssociation.get_package_ids_for_project( validated_data_dict['project_id']) pkg_list = [] if pkg_id_list is not None: # for each package id, get the package dict and append to list if # active for pkg_id in pkg_id_list: pkg = toolkit.get_action('package_show')(context, {'id': pkg_id}) if pkg['state'] == 'active': pkg_list.append(pkg) return pkg_list
def package_project_list(context, data_dict): '''List projects associated with a package. :param package_id: id or name of the package :type package_id: string :rtype: list of dictionaries ''' toolkit.check_access('ckanext_package_project_list', context, data_dict) # validate the incoming data_dict validated_data_dict, errors = validate(data_dict, package_project_list_schema(), context) if errors: raise toolkit.ValidationError(errors) # get a list of project ids associated with the package id project_id_list = projectPackageAssociation.get_project_ids_for_package( validated_data_dict['package_id']) project_list = [] if project_id_list is not None: for project_id in project_id_list: project = toolkit.get_action('package_show')(context, { 'id': project_id }) project_list.append(project) return project_list
def edit(self, id=None, data=None, errors=None, error_summary=None): context = {'save': 'save' in request.params, 'schema': self._edit_form_to_db_schema(), 'model': model, 'session': model.Session, 'user': c.user, 'auth_user_obj': c.userobj } if id is None: if c.userobj: id = c.userobj.id else: abort(400, _('No user specified')) data_dict = {'id': id} try: check_access('user_update', context, data_dict) except NotAuthorized: abort(403, _('Unauthorized to edit a user.')) if (context['save']) and not data: return self._save_edit(id, context) try: old_data = get_action('user_show')(context, data_dict) schema = self._db_to_edit_form_schema() if schema: old_data, errors = \ dictization_functions.validate(old_data, schema, context) c.display_name = old_data.get('display_name') c.user_name = old_data.get('name') data = data or old_data except NotAuthorized: abort(403, _('Unauthorized to edit user %s') % '') except NotFound: abort(404, _('User not found')) user_obj = context.get('user_obj') if not (authz.is_sysadmin(c.user) or c.user == user_obj.name): abort(403, _('User %s not authorized to edit %s') % (str(c.user), id)) errors = errors or {} vars = {'data': data, 'errors': errors, 'error_summary': error_summary} self._setup_template_variables({'model': model, 'session': model.Session, 'user': c.user}, data_dict) c.is_myself = True c.show_email_notifications = asbool( config.get('ckan.activity_streams_email_notifications')) c.form = render(self.edit_user_form, extra_vars=vars) return render('user/edit.html')
def resource_update(context, data_dict): model = context["model"] session = context["session"] user = context["user"] id = data_dict["id"] schema = context.get("schema") or default_update_resource_schema() model.Session.remove() resource = model.Resource.get(id) context["resource"] = resource if not resource: logging.error("Could not find resource " + id) raise NotFound(_("Resource was not found.")) check_access("resource_update", context, data_dict) data, errors = validate(data_dict, schema, context) if errors: model.Session.rollback() raise ValidationError(errors, resource_error_summary(errors)) rev = model.repo.new_revision() rev.author = user if "message" in context: rev.message = context["message"] else: rev.message = _(u"REST API: Update object %s") % data.get("name", "") resource = resource_dict_save(data, context) if not context.get("defer_commit"): model.repo.commit() return resource_dictize(resource, context)
def test_3_tag_schema_allows_spaces(self): """Asserts that a tag name with space is valid""" ignored = "" data = {"name": u"with space", "revision_timestamp": ignored, "state": ignored} _, errors = validate(data, default_tags_schema(), self.context) assert not errors, str(errors)
def test_5_tag_schema_allows_capital_letters(self): """Asserts that tag names can have capital letters""" ignored = "" data = {"name": u"CAPITALS", "revision_timestamp": ignored, "state": ignored} _, errors = validate(data, default_tags_schema(), self.context) assert not errors, str(errors)
def read_ajax(self, id, revision=None): package_type = self._get_package_type(id) context = { "model": model, "session": model.Session, "user": c.user or c.author, "extras_as_string": True, "schema": self._form_to_db_schema(package_type=package_type), "revision_id": revision, } try: data = get_action("package_show")(context, {"id": id}) schema = self._db_to_form_schema(package_type=package_type) if schema: data, errors = validate(data, schema) except NotAuthorized: abort(401, _("Unauthorized to read package %s") % "") except NotFound: abort(404, _("Dataset not found")) ## hack as db_to_form schema should have this data["tag_string"] = ", ".join([tag["name"] for tag in data.get("tags", [])]) data.pop("tags") data = flatten_to_string_key(data) response.headers["Content-Type"] = "application/json;charset=utf-8" return json.dumps(data)
def dataset_is_valid(package): """ Check if given dataset is valid. Uses schema from plugin. Return true if dataset is valid. """ package_plugin = plugins.lookup_package_plugin(package['type']) _, errors = validate(package, package_plugin.update_package_schema(), {'model': model, 'session': model.Session, 'user': c.user}) return not bool(errors)
def get_package_name_validation_errors(package_name): data_dict = {'name': package_name} data, errors = validate(data_dict, schema, context) return errors.get('name', [])
def test_1_package_schema(self): pkg = (model.Session.query( model.Package).filter_by(name="annakarenina").first()) package_id = pkg.id result = package_dictize(pkg, self.context) self.remove_changable_columns(result) result["name"] = "anna2" # we need to remove these as they have been added del result["relationships_as_object"] del result["relationships_as_subject"] converted_data, errors = validate(result, default_create_package_schema(), self.context) expected_data = { "extras": [ { "key": u"genre", "value": u"romantic novel" }, { "key": u"original media", "value": u"book" }, ], "groups": [ { u"name": u"david", u"title": u"Dave's books" }, { u"name": u"roger", u"title": u"Roger's books" }, ], "license_id": u"other-open", "name": u"anna2", "type": u"dataset", "notes": u"Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n\nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n", "private": False, "resources": [ { "alt_url": u"alt123", "description": u'Full text. Needs escaping: " Umlaut: \xfc', "format": u"plain text", "hash": u"abc123", "size_extra": u"123", "url": u"http://datahub.io/download/x=1&y=2", }, { "alt_url": u"alt345", "description": u"Index of the novel", "format": u"JSON", "hash": u"def456", "size_extra": u"345", "url": u"http://datahub.io/index.json", }, ], "tags": [ { "name": u"Flexible \u30a1" }, { "name": u"russian" }, { "name": u"tolstoy" }, ], "title": u"A Novel By Tolstoy", "url": u"http://datahub.io", "version": u"0.7a", } assert converted_data == expected_data, pformat(converted_data) assert not errors, errors data = converted_data data["name"] = u"annakarenina" data.pop("title") data["resources"][0]["url"] = "fsdfafasfsaf" data["resources"][1].pop("url") converted_data, errors = validate(data, default_create_package_schema(), self.context) assert errors == { "name": [u"That URL is already in use."] }, pformat(errors) data["id"] = package_id data["name"] = "????jfaiofjioafjij" converted_data, errors = validate(data, default_update_package_schema(), self.context) assert errors == { "name": [ u"Must be purely lowercase alphanumeric (ascii) " "characters and these symbols: -_" ] }, pformat(errors)
def test_package_schema(self): group1 = factories.Group(title="Dave's books") group2 = factories.Group(title="Roger's books") first_name = factories.Dataset.stub().name second_name = factories.Dataset.stub().name expected_data = { "extras": [ {"key": u"genre", "value": u"romantic novel"}, {"key": u"original media", "value": u"book"}, ], "groups": [ {u"name": group1["name"], u"title": group1["title"]}, {u"name": group2["name"], u"title": group2["title"]}, ], "license_id": u"other-open", "name": first_name, "type": u"dataset", "notes": u"Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n\nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n", "private": False, "resources": [ { "alt_url": u"alt123", "description": u'Full text. Needs escaping: " Umlaut: \xfc', "format": u"plain text", "hash": u"abc123", "size_extra": u"123", "url": u"http://datahub.io/download/x=1&y=2", }, { "alt_url": u"alt345", "description": u"Index of the novel", "format": u"JSON", "hash": u"def456", "size_extra": u"345", "url": u"http://datahub.io/index.json", }, ], "tags": sorted([ {"name": factories.Tag.stub().name}, {"name": factories.Tag.stub().name}, {"name": factories.Tag.stub().name}, ], key=operator.itemgetter("name")), "title": u"A Novel By Tolstoy", "url": u"http://datahub.io", "version": u"0.7a", "relationships_as_subject": [], "relationships_as_object": [], } context = {"model": model, "session": model.Session} pkg = factories.Dataset.model(**expected_data) package_id = pkg.id result = package_dictize(pkg, context) self.remove_changable_columns(result) result["name"] = second_name expected_data["name"] = second_name converted_data, errors = validate( result, default_create_package_schema(), context ) assert converted_data == expected_data, pformat(converted_data) assert not errors, errors data = converted_data data["name"] = first_name data.pop("title") data["resources"][0]["url"] = "fsdfafasfsaf" data["resources"][1].pop("url") converted_data, errors = validate( data, default_create_package_schema(), context ) assert errors == {"name": [u"That URL is already in use."]}, pformat( errors ) data["id"] = package_id data["name"] = "????jfaiofjioafjij" converted_data, errors = validate( data, default_update_package_schema(), context ) assert errors == { "name": [ u"Must be purely lowercase alphanumeric (ascii) " "characters and these symbols: -_" ] }, pformat(errors)
def request_create(context, data_dict): '''Create new request data. :param sender_name: The name of the sender who request data. :type sender_name: string :param organization: The sender's organization. :type organization: string :param email_address: The sender's email_address. :type email_address: string :param message_content: The content of the message. :type message_content: string :param package_id: The id of the package the data belongs to. :type package_id: string :returns: the newly created request data :rtype: dictionary ''' check_access('requestdata_request_create', context, data_dict) data, errors = df.validate(data_dict, schema.request_create_schema(), context) if errors: raise toolkit.ValidationError(errors) sender_name = data.get('sender_name') organization = data.get('organization') email_address = data.get('email_address') message_content = data.get('message_content') package_id = data.get('package_id') package = toolkit.get_action('package_show')(context, {'id': package_id}) sender_user_id = User.get(context['user']).id maintainers = package['maintainer'].split(',') data = { 'sender_name': sender_name, 'sender_user_id': sender_user_id, 'organization': organization, 'email_address': email_address, 'message_content': message_content, 'package_id': package_id } requestdata = ckanextRequestdata(**data) requestdata.save() maintainers_list = [] is_hdx = helpers.is_hdx_portal() for id in maintainers: try: if is_hdx: main_ids = toolkit.get_action('user_show')(context, {'id': id}) user = User.get(main_ids['id']) else: user = User.get(id) data = ckanextMaintainers() data.maintainer_id = user.id data.request_data_id = requestdata.id data.email = user.email maintainers_list.append(data) except NotFound: pass out = ckanextMaintainers.insert_all(maintainers_list, requestdata.id) return out
def action_user_update(context, data_dict): ''' Modified from CKAN: user_update Update a user account. Normal users can only update their own user accounts. Sysadmins can update any user account. For further parameters see ``user_create()``. :param id: the name or id of the user to update :type id: string :returns: the updated user account :rtype: dictionary ''' model = context['model'] user = context['user'] session = context['session'] schema = context.get('schema') or logic.schema.default_update_user_schema() # Modify the schema by adding translation related keys add_translation_modify_schema(schema) upload = uploader.Upload('user') upload.update_data_dict(data_dict, 'image_url', 'image_upload', 'clear_upload') ignore_missing = toolkit.get_validator('ignore_missing') convert_to_extras = toolkit.get_converter('convert_to_extras') schema['job_title'] = [ignore_missing, unicode, convert_to_extras] schema['telephone_number'] = [ignore_missing, unicode, convert_to_extras] schema['main_organization'] = [ignore_missing, unicode, convert_to_extras] schema['image_url'] = [ignore_missing, unicode, convert_to_extras] schema['linkedin'] = [ignore_missing, unicode, convert_to_extras] schema['facebook'] = [ignore_missing, unicode, convert_to_extras] schema['twitter'] = [ignore_missing, unicode, convert_to_extras] schema['blog'] = [ignore_missing, to_list_json, convert_to_extras] schema['www_page'] = [ignore_missing, to_list_json, convert_to_extras] # Add the localized keys for the localized fields to the schema schema = add_languages_modify(schema, _localized_fields) not_empty = toolkit.get_validator('not_empty') schema['fullname'] = [not_empty, unicode] id = logic.get_or_bust(data_dict, 'id') user_obj = model.User.get(id) context['user_obj'] = user_obj if user_obj is None: raise NotFound('User was not found.') # If the translations are not in the data_dict, the user has not added any translations or the user has deleted all translations. # Therefore, the translations are not sent with the POST so we need to empty and update the translations here. if 'translations' not in data_dict: data_dict['translations'] = [] toolkit.check_access('user_update', context, data_dict) data, errors = validate(data_dict, schema, context) if errors: session.rollback() raise ValidationError(errors) for extra in data['extras'] if 'extras' in data else []: user_obj.extras[extra['key']] = extra['value'] user = model_save.user_dict_save(data, context) activity_dict = { 'user_id': user.id, 'object_id': user.id, 'activity_type': 'changed user' } activity_create_context = { 'model': model, 'user': user, 'defer_commit': True, 'ignore_auth': True, 'session': session } toolkit.get_action('activity_create')(activity_create_context, activity_dict) # Attempt to update drupal user _update_drupal_user(context, data_dict) # TODO: Also create an activity detail recording what exactly changed in # the user. upload.upload(uploader.get_max_image_size()) if not context.get('defer_commit'): model.repo.commit() user_data = user_dictize(user, context) for key, value in user.extras.iteritems(): if key in user_data: log.warning( "Trying to override user data with extra variable '%s'", key) continue user_data[key] = value return user_data
def news_create(context, data_dict): '''Create a news. :param title: The title of the news. :type title: string :param description: Description of the news. :type description: string :param active: State of the news (optional). Default is true. :type active: boolean :param meta: Additional meta data for the news such as latitude/longitude etc. :type meta: string in JSON format :returns: the newly created news object :rtype: dictionary ''' log.info('News create: %r', data_dict) l.check_access('news_create', context, data_dict) data, errors = df.validate(data_dict, schema.news_create_schema(), context) if errors: raise t.ValidationError(errors) title = data.get('title') name = gen_news_name(title) content = data.get('content', u'') meta = data.get('meta', u'{}') expiration_date = data.get('expiration_date') image_url = data.get('image_url', u'') m = context.get('model') user_obj = m.User.get(context.get('user')) news = ckanextNews(title=title, name=name, content=content, meta=meta, expiration_date=expiration_date, image_url=image_url, creator_id=user_obj.id) news.save() out = news_dictize(news) # Send mail notification to all news subscribed users except the creator of the news # TODO: Email notifications should be sent asynchronous using celery tasks # TODO: Setup email server for testing mode send_email_condition = config.get('testing', False) if not send_email_condition: users = _get_action('news_mail_subscribed_users_show')( { 'ignore_auth': True }, {}) vars = { 'site_title_dk': config_option_show('ckan.site_title', 'da_DK'), 'site_title_en': config_option_show('ckan.site_title', 'en'), 'site_url': config.get('ckan.site_url'), 'news_item_title': out['title'], 'news_item_content': render_markdown(out['content'], True) } for u in users: if user_obj.id == u['subscriber_id']: continue u_obj = context['model'].User.get(u['subscriber_id']) if u_obj is None: continue vars['user_name'] = u_obj.name msg_body = render_jinja2('emails/news_published.txt', vars) msg_subject = render_jinja2('emails/news_published_subject.txt', vars) send_email(msg_body, u_obj.email, msg_subject) return out
def test_1_package_schema(self): pkg = model.Session.query( model.Package).filter_by(name='annakarenina').first() package_id = pkg.id result = package_dictize(pkg, self.context) self.remove_changable_columns(result) pprint(result) result['name'] = 'anna2' converted_data, errors = validate(result, default_package_schema(), self.context) pprint(errors) assert converted_data == { 'extras': [{ 'key': u'genre', 'value': u'"romantic novel"' }, { 'key': u'original media', 'value': u'"book"' }], 'groups': [{ 'name': u'david' }, { 'name': u'roger' }], 'license_id': u'other-open', 'name': u'anna2', 'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n', 'resources': [{ 'alt_url': u'alt123', 'description': u'Full text. Needs escaping: " Umlaut: \xfc', 'format': u'plain text', 'hash': u'abc123', 'size_extra': u'123', 'url': u'http://www.annakarenina.com/download/x=1&y=2' }, { 'alt_url': u'alt345', 'description': u'Index of the novel', 'format': u'json', 'hash': u'def456', 'size_extra': u'345', 'url': u'http://www.annakarenina.com/index.json' }], 'tags': [{ 'name': u'Flexible \u30a1' }, { 'name': u'russian' }, { 'name': u'tolstoy' }], 'title': u'A Novel By Tolstoy', 'url': u'http://www.annakarenina.com', 'version': u'0.7a' }, pformat(converted_data) assert not errors, errors data = converted_data data['name'] = u'annakarenina' data.pop("title") data["resources"][0]["url"] = 'fsdfafasfsaf' data["resources"][1].pop("url") converted_data, errors = validate(data, default_package_schema(), self.context) assert errors == { 'name': [u'That URL is already in use.'], #'resources': [{} # {'name': [u'That URL is already in use.']}] }, pformat(errors) data["id"] = package_id converted_data, errors = validate(data, default_package_schema(), self.context) assert errors == { #'resources': [{}, {'url': [u'Missing value']}] }, pformat(errors) data['name'] = '????jfaiofjioafjij' converted_data, errors = validate(data, default_package_schema(), self.context) assert errors == { 'name': [ u'Url must be purely lowercase alphanumeric (ascii) characters and these symbols: -_' ], #'resources': [{}, {'url': [u'Missing value']}] }, pformat(errors)
def test_group_schema(self): group = factories.Group.model() context = {"model": model, "session": model.Session} factories.Dataset.create_batch(2, groups=[{"name": group.name}]) data = group_dictize(group, context) # we don't want these here del data["groups"] del data["users"] del data["tags"] del data["extras"] converted_data, errors = validate(data, default_group_schema(), context) assert not errors group_pack = sorted(group.packages(), key=operator.attrgetter("id")) converted_data["packages"] = sorted(converted_data["packages"], key=operator.itemgetter("id")) expected = { "description": group.description, "id": group.id, "name": group.name, "is_organization": False, "type": u"group", "image_url": group.image_url, "image_display_url": group.image_url, "packages": sorted( [ { "id": group_pack[0].id, "name": group_pack[0].name, "title": group_pack[0].title, }, { "id": group_pack[1].id, "name": group_pack[1].name, "title": group_pack[1].title, }, ], key=operator.itemgetter("id"), ), "title": group.title, "approval_status": u"approved", } assert converted_data == expected, pformat(converted_data) data["packages"].sort(key=lambda x: x["id"]) data["packages"][0]["id"] = factories.Dataset.stub().name data["packages"][1].pop("id") data["packages"][1].pop("name") converted_data, errors = validate(data, default_group_schema(), context) assert errors == { "packages": [ { "id": [u"Not found: Dataset"] }, { "id": [u"Missing value"] }, ] }, pformat(errors)
def resource_create(context, data_dict): model = context['model'] user = context['user'] data, errors = validate(data_dict, default_resource_schema(), context)
def edit(self, id=None, data=None, errors=None, error_summary=None): context = { 'save': 'save' in request.params, 'schema': self._edit_form_to_db_schema(), 'model': model, 'session': model.Session, 'user': c.user, 'auth_user_obj': c.userobj } if id is None: if c.userobj: id = c.userobj.id else: abort(400, _('No user specified')) data_dict = {'id': id} try: check_access('user_update', context, data_dict) except NotAuthorized: abort(403, _('Unauthorized to edit a user.')) if (context['save']) and not data: return self._save_edit(id, context) try: old_data = get_action('user_show')(context, data_dict) schema = self._db_to_edit_form_schema() if schema: old_data, errors = \ dictization_functions.validate(old_data, schema, context) c.display_name = old_data.get('display_name') c.user_name = old_data.get('name') data = data or old_data except NotAuthorized: abort(403, _('Unauthorized to edit user %s') % '') except NotFound: abort(404, _('User not found')) user_obj = context.get('user_obj') if not (authz.is_sysadmin(c.user) or c.user == user_obj.name): abort(403, _('User %s not authorized to edit %s') % (str(c.user), id)) errors = errors or {} vars = {'data': data, 'errors': errors, 'error_summary': error_summary} self._setup_template_variables( { 'model': model, 'session': model.Session, 'user': c.user }, data_dict) c.is_myself = True c.show_email_notifications = asbool( config.get('ckan.activity_streams_email_notifications')) c.form = render(self.edit_user_form, extra_vars=vars) return render('user/edit.html')
def test_2_group_schema(self): group = model.Session.query(model.Group).first() data = group_dictize(group, self.context) # we don't want these here del data['groups'] del data['users'] del data['tags'] del data['extras'] converted_data, errors = validate(data, default_group_schema(), self.context) group_pack = sorted(group.packages(), key=lambda x: x.id) converted_data["packages"] = sorted(converted_data["packages"], key=lambda x: x["id"]) expected = { 'description': u'These are books that David likes.', 'id': group.id, 'name': u'david', 'is_organization': False, 'type': u'group', 'image_url': u'', 'image_display_url': u'', 'packages': sorted([{ 'id': group_pack[0].id, 'name': group_pack[0].name, 'title': group_pack[0].title }, { 'id': group_pack[1].id, 'name': group_pack[1].name, 'title': group_pack[1].title }], key=lambda x: x["id"]), 'title': u"Dave's books", 'approval_status': u'approved' } assert not errors assert converted_data == expected, pformat(converted_data) data["packages"].sort(key=lambda x: x["id"]) data["packages"][0]["id"] = 'fjdlksajfalsf' data["packages"][1].pop("id") data["packages"][1].pop("name") converted_data, errors = validate(data, default_group_schema(), self.context) assert errors == { 'packages': [{ 'id': [u'Not found: Dataset'] }, { 'id': [u'Missing value'] }] }, pformat(errors)
def related_update(self, context, data_dict): '''TAKEN FROM ACTION/UPDATE.PY Update a related item. You must be the owner of a related item to update it. For further parameters see ``related_create()``. :param id: the id of the related item to update :type id: string :returns: the updated related item :rtype: dictionary ''' model = context['model'] id = logic.get_or_bust(data_dict, "id") session = context['session'] schema = context.get( 'schema') or logic.schema.default_update_related_schema() related = model.Related.get(id) context["related"] = related if not related: logging.error('Could not find related ' + id) raise logic.NotFound(_('Item was not found.')) data, errors = df.validate(data_dict, schema, context) if errors: model.Session.rollback() raise logic.ValidationError(errors) related = dictization.model_save.related_dict_save(data, context) dataset_dict = None if 'package' in context: dataset = context['package'] dataset_dict = dictization.table_dictize(dataset, context) related_dict = dictization.model_dictize.related_dictize( related, context) activity_dict = { 'user_id': context['user'], 'object_id': related.id, 'activity_type': 'changed related item', } activity_dict['data'] = { 'related': related_dict, 'dataset': dataset_dict, } activity_create_context = { 'model': model, 'user': context['user'], 'defer_commit': True, 'ignore_auth': True, 'session': session } logic.get_action('activity_create')(activity_create_context, activity_dict) if not context.get('defer_commit'): model.repo.commit() return dictization.model_dictize.related_dictize(related, context)
def validate(self, config): import ckan.lib.navl.dictization_functions as df schema = self.into_schema() data, errors = df.validate(dict(config), schema) return data, errors
def test_simple(self): raise SkipTest() schema = { "name": [not_empty], "age": [ignore_missing, convert_int], "gender": [default("female")], } data = { "name": "fred", "age": "32", } converted_data, errors = validate(data, schema) assert not errors assert converted_data == { 'gender': 'female', 'age': 32, 'name': 'fred' }, converted_data data = { "name": "", "age": "dsa32", "extra": "extra", } converted_data, errors = validate(data, schema) assert errors == { ('age', ): [u'Please enter an integer value'], ('name', ): [u'Missing value'] }, errors assert converted_data == { 'gender': 'female', 'age': 'dsa32', 'name': '', '__extras': { 'extra': 'extra' } } data = { "name": "fred", "numbers": [{ "number": "13221312" }, { "number": "432423432", "code": "+44" }] } schema = { "name": [not_empty], "numbers": { "number": [convert_int], "code": [not_empty], "__extras": [ignore], } } converted_data, errors = validate(data, schema) assert errors == {('numbers', 0, 'code'): ['Missing value']}
def event_create(context, data_dict): '''Create an event. :param title: The title of the event. :type title: string :param description: Description of the event. :type description: string :param venue: Venue of the event. :type venue: string :param start: Start date of the event. :type start: string :param end: End date of the event. :type end: string :param active: State of the event (optional). Default is true. :type active: boolean :param meta: Additional meta data for the event such as latitude/longitude etc. :type meta: string in JSON format :returns: the newly created event :rtype: dictionary ''' log.info('Event create: %r', data_dict) l.check_access('event_create', context, data_dict) data, errors = df.validate(data_dict, schema.event_create_schema(), context) if errors: raise t.ValidationError(errors) title = data.get('title') name = gen_event_name(title) start = data.get('start') end = data.get('end') description = data.get('description', u'') venue = data.get('venue', u'') meta = data.get('meta', u'{}') active = data.get('active', False) m = context.get('model') user_obj = m.User.get(context.get('user')) event = ckanextEvent(title=title, name=name, description=description, start=start, end=end, venue=venue, meta=meta, active=active, creator_id=user_obj.id) event.save() out = event_dictize(event) return out
def get_tag_validation_errors(tag_name): data_dict = {'name': tag_name} data, errors = validate(data_dict, schema, context) return errors.get('name', [])
def test_simple(self): schema = { "name": [not_empty], "age": [ignore_missing, convert_int], "gender": [default("female")], } data = {"name": "fred", "age": "32"} converted_data, errors = validate(data, schema) assert not errors assert converted_data == { "gender": "female", "age": 32, "name": "fred", }, converted_data data = {"name": "", "age": "dsa32", "extra": "extra"} converted_data, errors = validate(data, schema) assert errors == { "age": [u"Please enter an integer value"], "name": [u"Missing value"], }, errors assert converted_data == { "gender": "female", "age": "dsa32", "name": "", "__extras": { "extra": "extra" }, } data = { "name": "fred", "numbers": [ { "number": "13221312" }, { "number": "432423432", "code": "+44" }, ], } schema = { "name": [not_empty], "numbers": { "number": [convert_int], "code": [not_empty], "__extras": [ignore], }, } converted_data, errors = validate(data, schema) assert errors == {"numbers": [{"code": [u"Missing value"]}]}
def get_package_version_validation_errors(package_version): data_dict = {'version': package_version} data, errors = validate(data_dict, schema, context) return errors.get('version', [])
def edit(self, id=None, data=None, errors=None, error_summary=None): context = { 'save': 'save' in request.params, 'schema': self._edit_form_to_db_schema(), 'model': model, 'session': model.Session, 'user': c.user, 'auth_user_obj': c.userobj } if id is None: if c.userobj: id = c.userobj.id else: abort(400, _('No user specified')) data_dict = {'id': id} try: check_access('user_update', context, data_dict) except NotAuthorized: abort(401, _('Unauthorized to edit a user.')) # Custom handling if user in organization action_ctx = context.copy() action_ctx['user'] = id c.in_organization = bool( logic.get_action('organization_list_for_user')( action_ctx, { 'permission': 'create_dataset' })) to_json = convert_to_json('about') not_empty = tk.get_validator('not_empty') if c.in_organization: context['schema'].update({ 'fullname': [not_empty, unicode], 'official_position': [not_empty, to_json], 'official_phone': [not_empty, to_json] }) # End of custom handling if (context['save']) and not data: return self._save_edit(id, context) try: if not data: data = get_action('user_show')(context, data_dict) schema = self._db_to_edit_form_schema() if schema: data, errors = df.validate(data, schema, context) c.display_name = data.get('display_name') c.user_name = data.get('name') except NotAuthorized: abort(401, _('Unauthorized to edit user %s') % '') except NotFound: abort(404, _('User not found')) user_obj = context.get('user_obj') if not (new_authz.is_sysadmin(c.user) or c.user == user_obj.name): abort(401, _('User %s not authorized to edit %s') % (str(c.user), id)) errors = errors or {} vars = {'data': data, 'errors': errors, 'error_summary': error_summary} self._setup_template_variables( { 'model': model, 'session': model.Session, 'user': c.user or c.author }, data_dict) c.is_myself = True c.show_email_notifications = h.asbool( config.get('ckan.activity_streams_email_notifications')) c.form = render(self.edit_user_form, extra_vars=vars) return render('user/edit.html')
def test_2_group_schema(self): group = model.Session.query(model.Group).first() data = group_dictize(group, self.context) # we don't want these here del data["groups"] del data["users"] del data["tags"] del data["extras"] converted_data, errors = validate(data, default_group_schema(), self.context) group_pack = sorted(group.packages(), key=lambda x: x.id) converted_data["packages"] = sorted(converted_data["packages"], key=lambda x: x["id"]) expected = { "description": u"These are books that David likes.", "id": group.id, "name": u"david", "is_organization": False, "type": u"group", "image_url": u"", "image_display_url": u"", "packages": sorted( [ { "id": group_pack[0].id, "name": group_pack[0].name, "title": group_pack[0].title, }, { "id": group_pack[1].id, "name": group_pack[1].name, "title": group_pack[1].title, }, ], key=lambda x: x["id"], ), "title": u"Dave's books", "approval_status": u"approved", } assert not errors assert converted_data == expected, pformat(converted_data) data["packages"].sort(key=lambda x: x["id"]) data["packages"][0]["id"] = "fjdlksajfalsf" data["packages"][1].pop("id") data["packages"][1].pop("name") converted_data, errors = validate(data, default_group_schema(), self.context) assert errors == { "packages": [ { "id": [u"Not found: Dataset"] }, { "id": [u"Missing value"] }, ] }, pformat(errors)