def func(key: FlattenKey, data: FlattenDataDict, errors: FlattenErrorDict,
context: Context):
new_tags = data.get(key)
if not new_tags:
return
if isinstance(new_tags, str):
new_tags = [new_tags]
# get current number of tags
n = 0
for k in data.keys():
if k[0] == 'tags':
n = max(n, k[1] + 1)
v = model.Vocabulary.get(vocab)
if not v:
raise df.Invalid(_('Tag vocabulary "%s" does not exist') % vocab)
context['vocabulary'] = v
for tag in new_tags:
validators.tag_in_vocabulary_validator(tag, context)
for num, tag in enumerate(new_tags):
data[('tags', num + n, 'name')] = tag
data[('tags', num + n, 'vocabulary_id')] = v.id
def convert(converter: Callable[..., Any], key: FlattenKey,
converted_data: FlattenDataDict, errors: FlattenErrorDict,
context: Context) -> None:
try:
nargs = converter.__code__.co_argcount
except AttributeError:
raise TypeError(f"{converter.__name__} cannot be used as validator "
"because it is not a user-defined function")
if nargs == 1:
params = (converted_data.get(key), )
elif nargs == 2:
params = (converted_data.get(key), context)
elif nargs == 4:
params = (key, converted_data, errors, context)
else:
raise TypeError("Wrong number of arguments for "
f"{converter.__name__}(expected 1, 2 or 4): {nargs}")
try:
value = converter(*params)
# 4-args version sets value internally
if nargs != 4:
converted_data[key] = value
return
except Invalid as e:
errors[key].append(e.error)
return
def ignore_not_package_admin(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict,
context: Context) -> Any:
'''Ignore if the user is not allowed to administer the package specified.'''
user = context.get('user')
if 'ignore_auth' in context:
return
if user and authz.is_sysadmin(user):
return
authorized = False
pkg = context.get('package')
if pkg:
try:
logic.check_access('package_change_state', context)
authorized = True
except logic.NotAuthorized:
authorized = False
if (user and pkg and authorized):
return
# allow_state_change in the context will allow the state to be changed
# FIXME is this the best way to cjeck for state only?
if key == ('state', ) and context.get('allow_state_change'):
return
data.pop(key)
def datasets_with_no_organization_cannot_be_private(key: FlattenKey,
data: FlattenDataDict,
errors: FlattenErrorDict,
context: Context) -> Any:
dataset_id = data.get(('id', ))
owner_org = data.get(('owner_org', ))
private = data[key] is True
check_passed = True
if not dataset_id and private and not owner_org:
# When creating a dataset, enforce it directly
check_passed = False
elif dataset_id and private and not owner_org:
# Check if the dataset actually has an owner_org, even if not provided
try:
dataset_dict = logic.get_action('package_show')({}, {
'id': dataset_id
})
if not dataset_dict.get('owner_org'):
check_passed = False
except logic.NotFound:
check_passed = False
if not check_passed:
errors[key].append(
_("Datasets with no organization can't be private."))
def callable(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context):
value = data.get(key)
other_value = data.get(key[:-1] + (other_key, ))
if (not value or value is missing and not other_value
or other_value is missing):
errors[key].append(_('Missing value'))
raise StopOnError
def free_tags_only(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context) -> Any:
tag_number = key[1]
if not data.get(('tags', tag_number, 'vocabulary_id')):
return
for k in list(data.keys()):
if k[0] == 'tags' and k[1] == tag_number:
del data[k]
def ignore_empty(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context) -> None:
value = data.get(key)
if value is missing or not value:
data.pop(key, None)
raise StopOnError
def owner_org_validator(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context) -> Any:
value = data.get(key)
if value is missing or value is None:
if not authz.check_config_permission('create_unowned_dataset'):
raise Invalid(_('An organization must be provided'))
data.pop(key, None)
raise df.StopOnError
model = context['model']
user = model.User.get(context['user'])
package = context.get('package')
if value == '':
if not authz.check_config_permission('create_unowned_dataset'):
raise Invalid(_('An organization must be provided'))
return
if (authz.check_config_permission('allow_dataset_collaborators')
and not authz.check_config_permission(
'allow_collaborators_to_change_owner_org')):
if package and user and not user.sysadmin:
is_collaborator = authz.user_is_collaborator_on_dataset(
user.id, package.id, ['admin', 'editor'])
if is_collaborator:
# User is a collaborator, check if it's also a member with
# edit rights of the current organization (redundant, but possible)
user_orgs = logic.get_action('organization_list_for_user')(
{
'ignore_auth': True
}, {
'id': user.id,
'permission': 'update_dataset'
})
user_is_org_member = package.owner_org in [
org['id'] for org in user_orgs
]
if data.get(
key) != package.owner_org and not user_is_org_member:
raise Invalid(
_('You cannot move this dataset to another organization'
))
group = model.Group.get(value)
if not group:
raise Invalid(_('Organization does not exist'))
group_id = group.id
if not package or (package and package.owner_org != group_id):
# This is a new dataset or we are changing the organization
if not context.get(u'ignore_auth', False) and (not user or not (
user.sysadmin or authz.has_user_permission_for_group_or_org(
group_id, user.name, 'create_dataset'))):
raise Invalid(_('You cannot add a dataset to this organization'))
data[key] = group_id
def ignore_not_sysadmin(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context) -> Any:
'''Ignore the field if user not sysadmin or ignore_auth in context.'''
user = context.get('user')
ignore_auth = context.get('ignore_auth')
if ignore_auth or (user and authz.is_sysadmin(user)):
return
data.pop(key)
def user_password_not_empty(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context) -> Any:
'''Only check if password is present if the user is created via action API.
If not, user_both_passwords_entered will handle the validation'''
# sysadmin may provide password_hash directly for importing users
if (data.get(('password_hash', ), missing) is not missing
and authz.is_sysadmin(context.get('user'))):
return
if not ('password1', ) in data and not ('password2', ) in data:
password = data.get(('password', ), None)
if not password:
errors[key].append(_('Missing value'))
def ignore(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context) -> NoReturn:
"""Remove the value from the input and skip the rest of validators.
.. code-block::
data, errors = tk.navl_validate(
{"hello": 1},
{"hello": [ignore]}
)
assert data == {}
"""
data.pop(key, None)
raise StopOnError
def rename_field(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context):
index = max([int(k[1]) for k in data.keys()
if len(k) == 3 and k[0] == new] + [-1])
for field_name in list(data.keys()):
if field_name[0] == old and data.get(field_name):
new_field_name = list(field_name)
new_field_name[0] = new
if len(new_field_name) > 1:
new_field_name[1] = int(new_field_name[1]) + index + 1
data[tuple(new_field_name)] = data[field_name]
data.pop(field_name)
def package_name_validator(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context) -> Any:
model = context['model']
session = context['session']
package = context.get('package')
query = session.query(model.Package.id).filter(
model.Package.name == data[key],
model.Package.state != State.DELETED,
)
if package:
package_id: Union[Optional[str], Missing] = package.id
else:
package_id = data.get(key[:-1] + ('id', ))
if package_id and package_id is not missing:
query = query.filter(model.Package.id != package_id)
if session.query(query.exists()).scalar():
errors[key].append(_('That URL is already in use.'))
value = data[key]
if len(value) < PACKAGE_NAME_MIN_LENGTH:
raise Invalid(
_('Name "%s" length is less than minimum %s') %
(value, PACKAGE_NAME_MIN_LENGTH))
if len(value) > PACKAGE_NAME_MAX_LENGTH:
raise Invalid(
_('Name "%s" length is more than maximum %s') %
(value, PACKAGE_NAME_MAX_LENGTH))
def untuplize_dict(tuplized_dict: FlattenDataDict) -> dict[str, Any]:
data_dict = {}
for key, value in tuplized_dict.items():
new_key = '__'.join([str(item) for item in key])
data_dict[new_key] = value
return data_dict
def remove_from_extras(data: FlattenDataDict, key: FlattenKey):
to_remove = []
for data_key, _data_value in data.items():
if (data_key[0] == 'extras' and data_key[1] == key):
to_remove.append(data_key)
for item in to_remove:
del data[item]
def not_empty(key: FlattenKey, data: FlattenDataDict, errors: FlattenErrorDict,
context: Context) -> None:
value = data.get(key)
if not value or value is missing:
errors[key].append(_('Missing value'))
raise StopOnError
def list_of_strings_or_lists(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context):
value = data.get(key)
if not isinstance(value, list):
raise df.Invalid('Not a list')
for x in value:
if not isinstance(x, str) and not isinstance(x, list):
raise df.Invalid('%s: %s' % ('Neither a string nor a list', x))
def list_of_strings(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context) -> Any:
value = data.get(key)
if not isinstance(value, list):
raise Invalid(_('Not a list'))
for x in value:
if not isinstance(x, str):
raise Invalid('%s: %s' % (_('Not a string'), x))
def unflatten(data: FlattenDataDict) -> dict[str, Any]:
'''Unflatten a simple dict whose keys are tuples.
e.g.
>>> unflatten(
{('name',): u'testgrp4',
('title',): u'',
('description',): u'',
('packages', 0, 'name'): u'testpkg',
('packages', 1, 'name'): u'testpkg',
('extras', 0, 'key'): u'packages',
('extras', 0, 'value'): u'["testpkg"]',
('extras', 1, 'key'): u'',
('extras', 1, 'value'): u'',
('state',): u'active'
('save',): u'Save Changes',
('cancel',): u'Cancel'})
{'name': u'testgrp4',
'title': u'',
'description': u'',
'packages': [{'name': u'testpkg'}, {'name': u'testpkg'}],
'extras': [{'key': u'packages', 'value': u'["testpkg"]'},
{'key': u'', 'value': u''}],
'state': u'active',
'save': u'Save Changes',
'cancel': u'Cancel'}
'''
unflattened: dict[str, Any] = {}
clean_lists: dict[int, Any] = {}
for flattend_key in sorted(data.keys(), key=flattened_order_key):
current_pos: Union[list[Any], dict[str, Any]] = unflattened
for key in flattend_key[:-1]:
try:
current_pos = current_pos[key]
except IndexError:
while True:
new_pos: Any = {}
assert isinstance(current_pos, list)
current_pos.append(new_pos)
if key < len(current_pos):
break
# skipped list indexes need to be removed before returning
clean_lists[id(current_pos)] = current_pos
current_pos = new_pos
except KeyError:
new_pos = []
current_pos[key] = new_pos
current_pos = new_pos
current_pos[flattend_key[-1]] = data[flattend_key]
for cl in clean_lists.values():
cl[:] = [i for i in cl if i]
return unflattened
def ignore_empty(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context) -> None:
"""Skip the rest of validators if the value is empty or missing.
.. code-block::
data, errors = tk.navl_validate(
{"hello": ""},
{"hello": [ignore_empty, isodate]}
)
assert data == {}
assert not errors
"""
value = data.get(key)
if value is missing or not value:
data.pop(key, None)
raise StopOnError
def user_both_passwords_entered(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict,
context: Context) -> Any:
password1 = data.get(('password1', ), None)
password2 = data.get(('password2', ), None)
if password1 is None or password1 == '' or \
password2 is None or password2 == '':
errors[('password', )].append(_('Please enter both passwords'))
def email_is_unique(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context) -> Any:
'''Validate email is unique'''
model = context['model']
session = context['session']
users = session.query(model.User) \
.filter(model.User.email == data[key]).all()
# is there is no users with this email it's free
if not users:
return
else:
# allow user to update their own email
for user in users:
if (user.name in (data.get(("name",)), data.get(("id",)))
or user.id == data.get(("id",))):
return
raise Invalid(
_('The email address \'{email}\' belongs to a registered user.').format(email=data[key]))
def user_both_passwords_entered(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict,
context: Context) -> Any:
"""Ensures that both password and password confirmation is not empty
"""
password1 = data.get(('password1', ), None)
password2 = data.get(('password2', ), None)
if password1 is None or password1 == '' or \
password2 is None or password2 == '':
errors[('password', )].append(_('Please enter both passwords'))
def if_empty_guess_format(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context) -> Any:
value = data[key]
resource_id = data.get(key[:-1] + ('id', ))
# if resource_id then an update
if (not value or value is Missing) and not resource_id:
url = data.get(key[:-1] + ('url', ), '')
if not url:
return
# Uploaded files have only the filename as url, so check scheme to
# determine if it's an actual url
parsed = urlparse(url)
if parsed.scheme and not parsed.path:
return
mimetype, _encoding = mimetypes.guess_type(url)
if mimetype:
data[key] = mimetype
def user_passwords_match(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context) -> Any:
password1 = data.get(('password1', ), None)
password2 = data.get(('password2', ), None)
if not password1 == password2:
errors[key].append(_('The passwords you entered do not match'))
else:
#Set correct password
data[('password', )] = password1
def ignore_missing(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context) -> None:
'''If the key is missing from the data, ignore the rest of the key's
schema.
By putting ignore_missing at the start of the schema list for a key,
you can allow users to post a dict without the key and the dict will pass
validation. But if they post a dict that does contain the key, then any
validators after ignore_missing in the key's schema list will be applied.
:raises ckan.lib.navl.dictization_functions.StopOnError: if ``data[key]``
is :py:data:`ckan.lib.navl.dictization_functions.missing` or ``None``
:returns: ``None``
'''
value = data.get(key)
if value is missing or value is None:
data.pop(key, None)
raise StopOnError
def convert_to_extras(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context) -> Any:
# Get the current extras index
current_indexes = [
k[1] for k in data.keys() if len(k) > 1 and k[0] == 'extras'
]
new_index = max(current_indexes) + 1 if current_indexes else 0
data[('extras', new_index, 'key')] = key[-1]
data[('extras', new_index, 'value')] = data[key]
def filter_fields_and_values_should_have_same_length(key: FlattenKey,
data: FlattenDataDict,
errors: FlattenErrorDict,
context: Context) -> Any:
convert_to_list_if_string = converters.convert_to_list_if_string
fields = convert_to_list_if_string(data.get(('filter_fields',), []))
values = convert_to_list_if_string(data.get(('filter_values',), []))
if len(fields) != len(values):
msg = _('"filter_fields" and "filter_values" should have the same length')
errors[('filter_fields',)].append(msg)
errors[('filter_values',)].append(msg)
def func(key: FlattenKey, data: FlattenDataDict, errors: FlattenErrorDict,
context: Context):
v = model.Vocabulary.get(vocab)
if not v:
raise df.Invalid(_('Tag vocabulary "%s" does not exist') % vocab)
tags = []
for k in data.keys():
if k[0] == 'tags':
if data[k].get('vocabulary_id') == v.id:
name = data[k].get('display_name', data[k]['name'])
tags.append(name)
data[key] = tags
def ignore_not_group_admin(key: FlattenKey, data: FlattenDataDict,
errors: FlattenErrorDict, context: Context) -> Any:
'''Ignore if the user is not allowed to administer for the group specified.'''
user = context.get('user')
if user and authz.is_sysadmin(user):
return
authorized = False
group = context.get('group')
if group:
try:
logic.check_access('group_change_state', context)
authorized = True
except logic.NotAuthorized:
authorized = False
if (user and group and authorized):
return
data.pop(key)
