def _parse_check(rule):
"""
Parse a single base check rule into an appropriate Check object.
"""
# Handle the special checks
if rule == '!':
return FalseCheck()
elif rule == '@':
return TrueCheck()
try:
kind, match = rule.split(':', 1)
except Exception:
LOG.exception(_("Failed to understand rule %s") % rule)
# If the rule is invalid, we'll fail closed
return FalseCheck()
# Find what implements the check
if kind in _checks:
return _checks[kind](kind, match)
elif None in _checks:
return _checks[None](kind, match)
else:
LOG.error(_("No handler for matches of kind %s") % kind)
return FalseCheck()
def validate_integer(value, property_name, min_val=None, max_val=None):
if isinstance(value, basestring):
try:
value = int(value)
except ValueError:
pass
value = __validate_type(value, property_name, (int, long), "Integer")
if min_val is not None and value < min_val:
raise exception.ValidationError(
_("'%(property_name)s' property value[%(property_value)s] is less "
"then min_value[%(min_value)s]."),
property_name=property_name,
property_value=value,
min_value=min_val
)
if max_val is not None and value > max_val:
raise exception.ValidationError(
_("'%(property_name)s' property value[%(property_value)s] is more "
"then max_value[%(max_value)s]."),
property_name=property_name,
property_value=value,
max_value=max_val
)
return value
def _parse_check(rule):
"""
Parse a single base check rule into an appropriate Check object.
"""
# Handle the special checks
if rule == '!':
return FalseCheck()
elif rule == '@':
return TrueCheck()
try:
kind, match = rule.split(':', 1)
except Exception:
LOG.exception(_("Failed to understand rule %s") % rule)
# If the rule is invalid, we'll fail closed
return FalseCheck()
# Find what implements the check
if kind in _checks:
return _checks[kind](kind, match)
elif None in _checks:
return _checks[None](kind, match)
else:
LOG.error(_("No handler for matches of kind %s") % kind)
return FalseCheck()
def parse_key_schema(cls, key_def_list_json):
hash_key_attr_name = None
range_key_attr_name = None
for key_def in key_def_list_json:
key_attr_name_json = key_def.pop(Props.ATTRIBUTE_NAME, None)
validation.validate_attr_name(key_attr_name_json)
key_type_json = key_def.pop(Props.KEY_TYPE, None)
if key_type_json == Values.KEY_TYPE_HASH:
if hash_key_attr_name is not None:
raise exception.ValidationError(
_("Only one 'HASH' key is allowed"))
hash_key_attr_name = key_attr_name_json
elif key_type_json == Values.KEY_TYPE_RANGE:
if range_key_attr_name is not None:
raise exception.ValidationError(
_("Only one 'RANGE' key is allowed"))
range_key_attr_name = key_attr_name_json
else:
raise exception.ValidationError(_(
"Only 'RANGE' or 'HASH' key types are allowed, but "
"'%(key_type)s' is found"),
key_type=key_type_json)
validation.validate_unexpected_props(key_def, "key_definition")
if hash_key_attr_name is None:
raise exception.ValidationError(_("HASH key is missing"))
if range_key_attr_name:
return (hash_key_attr_name, range_key_attr_name)
return (hash_key_attr_name, )
def parse_key_schema(cls, key_def_list_json):
hash_key_attr_name = None
range_key_attr_name = None
for key_def in key_def_list_json:
key_attr_name_json = key_def.pop(Props.ATTRIBUTE_NAME, None)
validation.validate_attr_name(key_attr_name_json)
key_type_json = key_def.pop(Props.KEY_TYPE, None)
if key_type_json == Values.KEY_TYPE_HASH:
if hash_key_attr_name is not None:
raise exception.ValidationError(
_("Only one 'HASH' key is allowed"))
hash_key_attr_name = key_attr_name_json
elif key_type_json == Values.KEY_TYPE_RANGE:
if range_key_attr_name is not None:
raise exception.ValidationError(
_("Only one 'RANGE' key is allowed"))
range_key_attr_name = key_attr_name_json
else:
raise exception.ValidationError(
_("Only 'RANGE' or 'HASH' key types are allowed, but "
"'%(key_type)s' is found"), key_type=key_type_json)
validation.validate_unexpected_props(key_def, "key_definition")
if hash_key_attr_name is None:
raise exception.ValidationError(_("HASH key is missing"))
if range_key_attr_name:
return (hash_key_attr_name, range_key_attr_name)
return (hash_key_attr_name,)
def parse_attribute_condition(cls, condition_type, condition_args,
condition_class=models.IndexedCondition):
actual_args_count = (
len(condition_args) if condition_args is not None else 0
)
if condition_type == Values.BETWEEN:
if actual_args_count != 2:
raise exception.ValidationError(
_("%(type)s condition type requires exactly 2 arguments, "
"but %(actual_args_count)s given"),
type=condition_type,
actual_args_count=actual_args_count
)
if condition_args[0].attr_type != condition_args[1].attr_type:
raise exception.ValidationError(
_("%(type)s condition type requires arguments of the "
"same type, but different types given"),
type=condition_type,
)
return [
condition_class.ge(condition_args[0]),
condition_class.le(condition_args[1])
]
if condition_type == Values.BEGINS_WITH:
first_condition = condition_class(
condition_class.CONDITION_TYPE_GREATER_OR_EQUAL,
condition_args
)
condition_arg = first_condition.arg
if condition_arg.is_number:
raise exception.ValidationError(
_("%(condition_type)s condition type is not allowed for"
"argument of the %(argument_type)s type"),
condition_type=condition_type,
argument_type=condition_arg.attr_type.type
)
first_value = condition_arg.decoded_value
chr_fun = unichr if isinstance(first_value, unicode) else chr
second_value = first_value[:-1] + chr_fun(ord(first_value[-1]) + 1)
second_condition = condition_class.le(
models.AttributeValue(
condition_arg.attr_type, decoded_value=second_value
)
)
return [first_condition, second_condition]
return [condition_class(condition_type, condition_args)]
def parse_local_secondary_index(cls, local_secondary_index_json):
key_attrs_json = local_secondary_index_json.pop(Props.KEY_SCHEMA, None)
validation.validate_list(key_attrs_json, Props.KEY_SCHEMA)
key_attrs_for_projection = cls.parse_key_schema(key_attrs_json)
hash_key = key_attrs_for_projection[0]
try:
range_key = key_attrs_for_projection[1]
except IndexError:
raise exception.ValidationError(
_("Range key in index wasn't specified"))
index_name = local_secondary_index_json.pop(Props.INDEX_NAME, None)
validation.validate_index_name(index_name)
projection_json = local_secondary_index_json.pop(Props.PROJECTION,
None)
validation.validate_object(projection_json, Props.PROJECTION)
validation.validate_unexpected_props(
local_secondary_index_json, "local_secondary_index"
)
projection_type = projection_json.pop(
Props.PROJECTION_TYPE, Values.PROJECTION_TYPE_INCLUDE
)
if projection_type == Values.PROJECTION_TYPE_ALL:
projected_attrs = None
elif projection_type == Values.PROJECTION_TYPE_KEYS_ONLY:
projected_attrs = tuple()
elif projection_type == Values.PROJECTION_TYPE_INCLUDE:
projected_attrs = projection_json.pop(
Props.NON_KEY_ATTRIBUTES, None
)
else:
raise exception.ValidationError(
_("Only '%(pt_all)', '%(pt_ko)' of '%(pt_incl)' projection "
"types are allowed, but '%(projection_type)s' is found"),
pt_all=Values.PROJECTION_TYPE_ALL,
pt_ko=Values.PROJECTION_TYPE_KEYS_ONLY,
pt_incl=Values.PROJECTION_TYPE_INCLUDE,
projection_type=projection_type
)
validation.validate_unexpected_props(projection_json, Props.PROJECTION)
return index_name, models.IndexDefinition(
hash_key,
range_key,
projected_attrs
)
def parse_attribute_condition(cls,
condition_type,
condition_args,
condition_class=models.IndexedCondition):
actual_args_count = (len(condition_args)
if condition_args is not None else 0)
if condition_type == Values.BETWEEN:
if actual_args_count != 2:
raise exception.ValidationError(
_("%(type)s condition type requires exactly 2 arguments, "
"but %(actual_args_count)s given"),
type=condition_type,
actual_args_count=actual_args_count)
if condition_args[0].attr_type != condition_args[1].attr_type:
raise exception.ValidationError(
_("%(type)s condition type requires arguments of the "
"same type, but different types given"),
type=condition_type,
)
return [
condition_class.ge(condition_args[0]),
condition_class.le(condition_args[1])
]
if condition_type == Values.BEGINS_WITH:
first_condition = condition_class(
condition_class.CONDITION_TYPE_GREATER_OR_EQUAL,
condition_args)
condition_arg = first_condition.arg
if condition_arg.is_number:
raise exception.ValidationError(
_("%(condition_type)s condition type is not allowed for"
"argument of the %(argument_type)s type"),
condition_type=condition_type,
argument_type=condition_arg.attr_type.type)
first_value = condition_arg.decoded_value
chr_fun = unichr if isinstance(first_value, unicode) else chr
second_value = first_value[:-1] + chr_fun(ord(first_value[-1]) + 1)
second_condition = condition_class.le(
models.AttributeValue(condition_arg.attr_type,
decoded_value=second_value))
return [first_condition, second_condition]
return [condition_class(condition_type, condition_args)]
def __init__(self, attribute_type_map, key_attributes, index_def_map=None):
"""
:param attribute_type_map: attribute name to AttributeType mapping
:param key_attrs: list of key attribute names, contains partition key
(the first in list, required) attribute name and extra key
attribute names (the second and other list items, not
required)
:param index_def_map: index name to IndexDefinition mapping
"""
if index_def_map is None:
index_def_map = {}
for key_attr in key_attributes:
if key_attr not in attribute_type_map:
raise exception.ValidationError(
"Definition for attribute['%(attr_name)s'] wasn't found",
attr_name=key_attr
)
# Validate the primary hash and range key types, make sure them are
# scalar types
attr_type = attribute_type_map.get(key_attr, None)
if attr_type is not None and attr_type.collection_type is not None:
raise exception.ValidationError(
_("Type '%(prop_value)s' is not a scalar type"),
prop_value=attr_type['type'])
if len(key_attr) < 2 and index_def_map:
raise exception.ValidationError("Local secondary indexes are not "
"allowed for tables with hash "
"key only")
for index_name, index_def in index_def_map.iteritems():
if index_def.alt_hash_key_attr != key_attributes[0]:
msg = _("Hash key of index '%(index_name)s' must "
"be the same as primary key's hash key.")
raise exception.ValidationError(msg, index_name=index_name)
if index_def.alt_range_key_attr not in attribute_type_map:
raise exception.ValidationError(
"Definition for attribute['%(attr_name)s'] wasn't found",
attr_name=index_def.alt_range_key_attr
)
super(TableSchema, self).__init__(
attribute_type_map=attribute_type_map,
key_attributes=key_attributes,
index_def_map=index_def_map)
def __decode_value(cls, attr_type, encoded_value):
decoded_value = None
if decoded_value is not None:
return decoded_value
collection_type = attr_type.collection_type
if collection_type is None:
decoded_value = cls.__decode_single_value(attr_type.type,
encoded_value)
elif collection_type == AttributeType.COLLECTION_TYPE_MAP:
if isinstance(encoded_value, dict):
res_dict = dict()
key_type = attr_type.key_type
value_type = attr_type.value_type
for key, value in encoded_value.iteritems():
res_dict[cls.__decode_single_value(key_type, key)] = (
cls.__decode_single_value(value_type, value)
)
decoded_value = res_dict
elif collection_type == AttributeType.COLLECTION_TYPE_SET:
element_type = attr_type.element_type
res = blist.sortedset()
for val in encoded_value:
res.add(cls.__decode_single_value(element_type, val))
decoded_value = res
if decoded_value is None:
raise exception.ValidationError(
_("Can't recognize attribute value '%(value)s'"
"of type %(type)s"),
type=attr_type, value=json.dumps(encoded_value)
)
return decoded_value
def bool_from_string(subject, strict=False, default=False):
"""Interpret a string as a boolean.
A case-insensitive match is performed such that strings matching 't',
'true', 'on', 'y', 'yes', or '1' are considered True and, when
`strict=False`, anything else returns the value specified by 'default'.
Useful for JSON-decoded stuff and config file parsing.
If `strict=True`, unrecognized values, including None, will raise a
ValueError which is useful when parsing values passed in from an API call.
Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
"""
if not isinstance(subject, six.string_types):
subject = six.text_type(subject)
lowered = subject.strip().lower()
if lowered in TRUE_STRINGS:
return True
elif lowered in FALSE_STRINGS:
return False
elif strict:
acceptable = ', '.join(
"'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
msg = _("Unrecognized value '%(val)s', acceptable values are:"
" %(acceptable)s") % {'val': subject,
'acceptable': acceptable}
raise ValueError(msg)
else:
return default
def validate_index_name(value):
validate_string(value, "index name")
if not INDEX_NAME_PATTERN.match(value):
raise exception.ValidationError(
_("Wrong index name '%(prop_value)s' found"), prop_value=value)
return value
def _raise_condition_schema_mismatch(condition_map, table_info):
raise exception.ValidationError(
_("Specified query conditions %(indexed_condition_map)s "
"don't match table schema: %(table_schema)s"),
indexed_condition_map=condition_map,
table_schema=table_info.schema
)
def deprecated(self, msg, *args, **kwargs):
stdmsg = "Deprecated: %s"
if CONF.fatal_deprecations:
self.critical(_LC(stdmsg) % msg, *args, **kwargs)
raise DeprecatedConfig(msg=_(stdmsg))
else:
self.warn(_LW(stdmsg) % msg, *args, **kwargs)
def validate_table_name(value):
validate_string(value, "table name")
if not TABLE_NAME_PATTERN.match(value):
raise exception.ValidationError(
_("Wrong table name '%(prop_value)s' found"), prop_value=value)
return value
def parse_batch_write_request_items(cls, request_items_json):
request_map = {}
for table_name, request_list_json in request_items_json.iteritems():
validation.validate_table_name(table_name)
validation.validate_list_of_objects(request_list_json, table_name)
request_list_for_table = []
for request_json in request_list_json:
for request_type, request_body in request_json.iteritems():
validation.validate_string(request_type, "request_type")
if request_type == Props.REQUEST_PUT:
validation.validate_object(request_body, request_type)
item = request_body.pop(Props.ITEM, None)
validation.validate_object(item, Props.ITEM)
validation.validate_unexpected_props(
request_body, request_type)
request_list_for_table.append(
models.WriteItemRequest.put(
cls.parse_item_attributes(item)))
elif request_type == Props.REQUEST_DELETE:
validation.validate_object(request_body, request_type)
key = request_body.pop(Props.KEY, None)
validation.validate_object(key, Props.KEY)
validation.validate_unexpected_props(
request_body, request_type)
request_list_for_table.append(
models.WriteItemRequest.delete(
cls.parse_item_attributes(key)))
else:
raise exception.ValidationError(
_("Unsupported request type found: "
"%(request_type)s"),
request_type=request_type)
request_map[table_name] = request_list_for_table
return request_map
def deprecated(self, msg, *args, **kwargs):
stdmsg = "Deprecated: %s"
if CONF.fatal_deprecations:
self.critical(_LC(stdmsg) % msg, *args, **kwargs)
raise DeprecatedConfig(msg=_(stdmsg))
else:
self.warn(_LW(stdmsg) % msg, *args, **kwargs)
def process_request(self, req):
"""
Extract any authentication information in the request and
construct an appropriate context from it.
"""
# Use the default empty context, with admin turned on for
# backwards compatibility
user_id = req.headers.get('X-User-Id', None)
tenant_id = req.headers.get('X-Tenant-Id', None)
roles = req.headers.get('X-Roles', None)
service_catalog = None
if req.headers.get('X_SERVICE_CATALOG') is not None:
try:
catalog_header = req.headers.get('X_SERVICE_CATALOG')
service_catalog = json.loads(catalog_header)
except ValueError:
raise webob.exc.HTTPInternalServerError(
explanation=_('Invalid service catalog json.'))
auth_token = req.headers.get('X-Auth-Token', None)
req.environ['magnetodb.context'] = self.make_context(
is_admin=True,
auth_token=auth_token,
user_id=user_id,
tenant_id=tenant_id,
roles=roles,
service_catalog=service_catalog
)
def bool_from_string(subject, strict=False, default=False):
"""Interpret a string as a boolean.
A case-insensitive match is performed such that strings matching 't',
'true', 'on', 'y', 'yes', or '1' are considered True and, when
`strict=False`, anything else returns the value specified by 'default'.
Useful for JSON-decoded stuff and config file parsing.
If `strict=True`, unrecognized values, including None, will raise a
ValueError which is useful when parsing values passed in from an API call.
Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
"""
if not isinstance(subject, six.string_types):
subject = six.text_type(subject)
lowered = subject.strip().lower()
if lowered in TRUE_STRINGS:
return True
elif lowered in FALSE_STRINGS:
return False
elif strict:
acceptable = ', '.join("'%s'" % s
for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
msg = _("Unrecognized value '%(val)s', acceptable values are:"
" %(acceptable)s") % {
'val': subject,
'acceptable': acceptable
}
raise ValueError(msg)
else:
return default
def parse_typed_attr_value(cls, typed_attr_value_json):
if len(typed_attr_value_json) != 1:
raise exception.ValidationError(
_("Can't recognize attribute typed value format: '%(attr)s'"),
attr=json.dumps(typed_attr_value_json))
(attr_type_json, attr_value_json) = (typed_attr_value_json.popitem())
return models.AttributeValue(attr_type_json, attr_value_json)
def __init__(self, select_type, attributes=None):
if select_type not in self._allowed_types:
raise exception.ValidationError(
_("Select type '%(select_type)s' isn't allowed"),
select_type=select_type
)
if attributes is not None:
if select_type != self.SELECT_TYPE_SPECIFIC:
raise exception.ValidationError(
_("Attribute list is only expected with select_type "
"'%(select_type)s'"),
select_type=self.SELECT_TYPE_SPECIFIC
)
super(SelectType, self).__init__(type=select_type,
attributes=attributes)
def validate_attr_name(value):
validate_string(value, "attribute name")
if not ATTRIBUTE_NAME_PATTERN.match(value):
raise exception.ValidationError(
_("Wrong attribute name '%(prop_value)s' found"),
prop_value=value
)
def execute_write_batch(self, context, write_request_map):
write_request_list_to_send = []
for table_name, write_request_list in write_request_map.iteritems():
table_info = self._table_info_repo.get(context, table_name)
requested_keys = set()
for req in write_request_list:
self._validate_table_is_active(table_info)
if req.is_put:
self._validate_table_schema(table_info,
req.attribute_map,
keys_only=False)
else:
self._validate_table_schema(table_info, req.attribute_map)
key_values = self._key_values(table_info, req.attribute_map)
keys = tuple(key_values)
if keys in requested_keys:
raise exception.ValidationError(_(
"Can't execute request: "
"More than one operation requested"
" for item with keys %(keys)s"
" in table '%(table_name)s'"),
table_name=table_info.name,
keys=keys)
requested_keys.add(keys)
write_request_list_to_send.append((table_info, req))
future_result_list = []
for i in xrange(0, len(write_request_list_to_send),
self._batch_chunk_size):
req_list = (write_request_list_to_send[i:i +
self._batch_chunk_size])
future_result_list.append(
self._batch_write_async(context, req_list))
unprocessed_items = {}
for future_result in future_result_list:
unprocessed_request_list = future_result.result()
for (table_info, write_request) in unprocessed_request_list:
table_name = table_info.name
tables_unprocessed_items = (unprocessed_items.get(
table_name, None))
if tables_unprocessed_items is None:
tables_unprocessed_items = []
unprocessed_items[table_name] = tables_unprocessed_items
tables_unprocessed_items.append(write_request)
return unprocessed_items
def validate_table_name(value):
validate_string(value, "table name")
if not TABLE_NAME_PATTERN.match(value):
raise exception.ValidationError(
_("Wrong table name '%(prop_value)s' found"),
prop_value=value
)
return value
def __call__(self, request):
"""WSGI method that controls (de)serialization and method dispatch."""
try:
action, action_args, accept = self.deserialize_request(request)
except exception.InvalidContentType:
msg = _("Unsupported Content-Type")
return webob.exc.HTTPUnsupportedMediaType(explanation=msg)
except exception.MalformedRequestBody:
msg = _("Malformed request body")
return webob.exc.HTTPBadRequest(explanation=msg)
action_result = self.execute_action(action, request, **action_args)
try:
return self.serialize_response(action, action_result, accept)
# return unserializable result (typically a webob exc)
except Exception:
return action_result
def validate_index_name(value):
validate_string(value, "index name")
if not INDEX_NAME_PATTERN.match(value):
raise exception.ValidationError(
_("Wrong index name '%(prop_value)s' found"),
prop_value=value
)
return value
def _validate_table_is_active(table_info):
if table_info.status != models.TableMeta.TABLE_STATUS_ACTIVE:
raise exception.ValidationError(
_("Can't execute request: "
"Table '%(table_name)s' status '%(table_status)s' "
"isn't %(expected_status)s"),
table_name=table_info.name, table_status=table_info.status,
expected_status=models.TableMeta.TABLE_STATUS_ACTIVE
)
def _validate_table_is_active(table_info):
if table_info.status != models.TableMeta.TABLE_STATUS_ACTIVE:
raise exception.ValidationError(
_("Can't execute request: "
"Table '%(table_name)s' status '%(table_status)s' "
"isn't %(expected_status)s"),
table_name=table_info.name,
table_status=table_info.status,
expected_status=models.TableMeta.TABLE_STATUS_ACTIVE)
def string_to_bytes(text, unit_system='IEC', return_int=False):
"""Converts a string into an float representation of bytes.
The units supported for IEC ::
Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it)
KB, KiB, MB, MiB, GB, GiB, TB, TiB
The units supported for SI ::
kb(it), Mb(it), Gb(it), Tb(it)
kB, MB, GB, TB
Note that the SI unit system does not support capital letter 'K'
:param text: String input for bytes size conversion.
:param unit_system: Unit system for byte size conversion.
:param return_int: If True, returns integer representation of text
in bytes. (default: decimal)
:returns: Numerical representation of text in bytes.
:raises ValueError: If text has an invalid value.
"""
try:
base, reg_ex = UNIT_SYSTEM_INFO[unit_system]
except KeyError:
msg = _('Invalid unit system: "%s"') % unit_system
raise ValueError(msg)
match = reg_ex.match(text)
if match:
magnitude = float(match.group(1))
unit_prefix = match.group(2)
if match.group(3) in ['b', 'bit']:
magnitude /= 8
else:
msg = _('Invalid string format: %s') % text
raise ValueError(msg)
if not unit_prefix:
res = magnitude
else:
res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix])
if return_int:
return int(math.ceil(res))
return res
def parse_local_secondary_index(cls, local_secondary_index_json):
key_attrs_json = local_secondary_index_json.pop(Props.KEY_SCHEMA, None)
validation.validate_list(key_attrs_json, Props.KEY_SCHEMA)
key_attrs_for_projection = cls.parse_key_schema(key_attrs_json)
hash_key = key_attrs_for_projection[0]
try:
range_key = key_attrs_for_projection[1]
except IndexError:
raise exception.ValidationError(
_("Range key in index wasn't specified"))
index_name = local_secondary_index_json.pop(Props.INDEX_NAME, None)
validation.validate_index_name(index_name)
projection_json = local_secondary_index_json.pop(
Props.PROJECTION, None)
validation.validate_object(projection_json, Props.PROJECTION)
validation.validate_unexpected_props(local_secondary_index_json,
"local_secondary_index")
projection_type = projection_json.pop(Props.PROJECTION_TYPE,
Values.PROJECTION_TYPE_INCLUDE)
if projection_type == Values.PROJECTION_TYPE_ALL:
projected_attrs = None
elif projection_type == Values.PROJECTION_TYPE_KEYS_ONLY:
projected_attrs = tuple()
elif projection_type == Values.PROJECTION_TYPE_INCLUDE:
projected_attrs = projection_json.pop(Props.NON_KEY_ATTRIBUTES,
None)
else:
raise exception.ValidationError(
_("Only '%(pt_all)', '%(pt_ko)' of '%(pt_incl)' projection "
"types are allowed, but '%(projection_type)s' is found"),
pt_all=Values.PROJECTION_TYPE_ALL,
pt_ko=Values.PROJECTION_TYPE_KEYS_ONLY,
pt_incl=Values.PROJECTION_TYPE_INCLUDE,
projection_type=projection_type)
validation.validate_unexpected_props(projection_json, Props.PROJECTION)
return index_name, models.IndexDefinition(hash_key, range_key,
projected_attrs)
def string_to_bytes(text, unit_system='IEC', return_int=False):
"""Converts a string into an float representation of bytes.
The units supported for IEC ::
Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it)
KB, KiB, MB, MiB, GB, GiB, TB, TiB
The units supported for SI ::
kb(it), Mb(it), Gb(it), Tb(it)
kB, MB, GB, TB
Note that the SI unit system does not support capital letter 'K'
:param text: String input for bytes size conversion.
:param unit_system: Unit system for byte size conversion.
:param return_int: If True, returns integer representation of text
in bytes. (default: decimal)
:returns: Numerical representation of text in bytes.
:raises ValueError: If text has an invalid value.
"""
try:
base, reg_ex = UNIT_SYSTEM_INFO[unit_system]
except KeyError:
msg = _('Invalid unit system: "%s"') % unit_system
raise ValueError(msg)
match = reg_ex.match(text)
if match:
magnitude = float(match.group(1))
unit_prefix = match.group(2)
if match.group(3) in ['b', 'bit']:
magnitude /= 8
else:
msg = _('Invalid string format: %s') % text
raise ValueError(msg)
if not unit_prefix:
res = magnitude
else:
res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix])
if return_int:
return int(math.ceil(res))
return res
def _wrap(*args, **kw):
try:
return f(*args, **kw)
except Exception as e:
if not isinstance(e, Error):
# exc_type, exc_value, exc_traceback = sys.exc_info()
logging.exception(_('Uncaught exception'))
# logging.error(traceback.extract_stack(exc_traceback))
raise Error(str(e))
raise
def _wrap(*args, **kw):
try:
return f(*args, **kw)
except Exception as e:
if not isinstance(e, Error):
# exc_type, exc_value, exc_traceback = sys.exc_info()
logging.exception(_('Uncaught exception'))
# logging.error(traceback.extract_stack(exc_traceback))
raise Error(str(e))
raise
def _validate_table_schema(table_info, attribute_map, keys_only=True,
index_name=None):
schema_key_attributes = table_info.schema.key_attributes
schema_attribute_type_map = table_info.schema.attribute_type_map
key_attribute_names_to_find = set(schema_key_attributes)
if index_name is not None:
key_attribute_names_to_find.add(
table_info.schema.index_def_map[index_name].alt_range_key_attr
)
if keys_only and (
len(key_attribute_names_to_find) != len(attribute_map)):
raise exception.ValidationError(
_("Specified key: %(key_attributes)s doesn't match expected "
"key attributes set: %(expected_key_attributes)s"),
key_attributes=attribute_map,
expected_key_attributes=key_attribute_names_to_find
)
for attr_name, typed_attr_value in attribute_map.iteritems():
schema_attr_type = schema_attribute_type_map.get(attr_name, None)
if schema_attr_type is None:
continue
key_attribute_names_to_find.discard(attr_name)
if schema_attr_type != typed_attr_value.attr_type:
raise exception.ValidationError(
_("Attribute: '%(attr_name)s' of type: '%(attr_type)s' "
"doesn't match table schema expected attribute type: "
"'%(expected_attr_type)s'"),
attr_name=attr_name,
attr_type=typed_attr_value.attr_type.type,
expected_attr_type=schema_attr_type.type
)
if key_attribute_names_to_find:
raise exception.ValidationError(
_("Couldn't find expected key attributes: "
"'%(expected_key_attributes)s'"),
expected_key_attributes=key_attribute_names_to_find
)
def __init__(self, type, args):
allowed_arg_count = self._allowed_types_to_arg_count_map.get(type,
None)
if allowed_arg_count is None:
raise exception.ValidationError(
_("%(condition_class)s of type['%(type)s'] is not allowed"),
condition_class=self.__class__.__name__, type=type)
actual_arg_count = len(args) if args is not None else 0
if (actual_arg_count < allowed_arg_count[0] or
actual_arg_count > allowed_arg_count[1]):
if allowed_arg_count[0] == allowed_arg_count[1]:
raise exception.ValidationError(
_("%(condition_class)s of type['%(type)s'] requires "
"exactly %(allowed_arg_count)s arguments, "
"but %(actual_arg_count)s found"),
condition_class=self.__class__.__name__, type=type,
allowed_arg_count=allowed_arg_count[0],
actual_arg_count=actual_arg_count
)
else:
raise exception.ValidationError(
_("%(condition_class)s of type['%(type)s'] requires from "
"%(min_args_allowed)s to %(max_args_allowed)s arguments "
"provided, but %(actual_arg_count)s found"),
condition_class=self.__class__.__name__, type=type,
min_args_allowed=allowed_arg_count[0],
max_args_allowed=allowed_arg_count[1],
actual_arg_count=actual_arg_count
)
if args is not None and type in self._types_with_only_primitive_arg:
for arg in args:
if arg.attr_type.collection_type is not None:
raise exception.ValidationError(
_("%(condition_class)s of type['%(type)s'] allows "
"only primitive arguments"),
condition_class=self.__class__.__name__, type=type
)
super(Condition, self).__init__(type=type, args=args)
def parse_typed_attr_value(cls, typed_attr_value_json):
if len(typed_attr_value_json) != 1:
raise exception.ValidationError(
_("Can't recognize attribute typed value format: '%(attr)s'"),
attr=json.dumps(typed_attr_value_json)
)
(attr_type_json, attr_value_json) = (
typed_attr_value_json.popitem()
)
return models.AttributeValue(attr_type_json, attr_value_json)
def __init__(self, type):
"""
:param type: one of available return values type
"""
if type not in self._allowed_types:
raise exception.ValidationError(
_("Return values type '%(type)s' isn't allowed"),
type=type
)
super(DeleteReturnValuesType, self).__init__(type=type)
class LogConfigError(Exception):
message = _('Error loading logging config %(log_config)s: %(err_msg)s')
def __init__(self, log_config, err_msg):
self.log_config = log_config
self.err_msg = err_msg
def __str__(self):
return self.message % dict(log_config=self.log_config,
err_msg=self.err_msg)
def parse_local_secondary_indexes(cls, local_secondary_index_list_json):
res = {}
for index_json in local_secondary_index_list_json:
index_name, index_def = (
cls.parse_local_secondary_index(index_json))
res[index_name] = index_def
if len(res) < len(local_secondary_index_list_json):
raise exception.ValidationError(
_("Two or more indexes with the same name"))
return res
def parse_expected_attribute_conditions(
cls, expected_attribute_conditions_json):
expected_attribute_conditions = {}
for (attr_name_json, condition_json) in (
expected_attribute_conditions_json.iteritems()):
validation.validate_attr_name(attr_name_json)
validation.validate_object(condition_json, attr_name_json)
if len(condition_json) != 1:
raise exception.ValidationError(
_("Can't recognize attribute expected condition format: "
"'%(attr)s'"),
attr=json.dumps(condition_json)
)
(condition_type, condition_value) = condition_json.popitem()
validation.validate_string(condition_type, "condition type")
if condition_type == Props.VALUE:
validation.validate_object(condition_value, Props.VALUE)
expected_attribute_conditions[attr_name_json] = [
models.ExpectedCondition.eq(
cls.parse_typed_attr_value(condition_value)
)
]
elif condition_type == Props.EXISTS:
validation.validate_boolean(condition_value, Props.EXISTS)
expected_attribute_conditions[attr_name_json] = [
models.ExpectedCondition.not_null() if condition_value else
models.ExpectedCondition.null()
]
else:
raise exception.ValidationError(
_("Unsupported condition type found: %(condition_type)s"),
condition_type=condition_type
)
return expected_attribute_conditions
def _validate_table_schema(table_info,
attribute_map,
keys_only=True,
index_name=None):
schema_key_attributes = table_info.schema.key_attributes
schema_attribute_type_map = table_info.schema.attribute_type_map
key_attribute_names_to_find = set(schema_key_attributes)
if index_name is not None:
key_attribute_names_to_find.add(
table_info.schema.index_def_map[index_name].alt_range_key_attr)
if keys_only and (len(key_attribute_names_to_find) !=
len(attribute_map)):
raise exception.ValidationError(
_("Specified key: %(key_attributes)s doesn't match expected "
"key attributes set: %(expected_key_attributes)s"),
key_attributes=attribute_map,
expected_key_attributes=key_attribute_names_to_find)
for attr_name, typed_attr_value in attribute_map.iteritems():
schema_attr_type = schema_attribute_type_map.get(attr_name, None)
if schema_attr_type is None:
continue
key_attribute_names_to_find.discard(attr_name)
if schema_attr_type != typed_attr_value.attr_type:
raise exception.ValidationError(
_("Attribute: '%(attr_name)s' of type: '%(attr_type)s' "
"doesn't match table schema expected attribute type: "
"'%(expected_attr_type)s'"),
attr_name=attr_name,
attr_type=typed_attr_value.attr_type.type,
expected_attr_type=schema_attr_type.type)
if key_attribute_names_to_find:
raise exception.ValidationError(
_("Couldn't find expected key attributes: "
"'%(expected_key_attributes)s'"),
expected_key_attributes=key_attribute_names_to_find)
def validate_unexpected_props(value, property_name):
if len(value) > 0:
if isinstance(value, dict):
value_str = json.dumps(value)
else:
value_str = str(value)
raise exception.ValidationError(_(
"Unexpected properties were found for '%(property_name)s': "
"%(unexpected_props)s"),
property_name=property_name,
unexpected_props=value_str)
return value
def __validate_type(value, property_name, py_type, json_type):
if value is None:
raise exception.ValidationError(_(
"Required property '%(property_name)s' wasn't found "
"or it's value is null"),
property_name=property_name)
if not isinstance(value, py_type):
raise exception.ValidationError(WRONG_TYPE_MSG,
property_name=property_name,
json_type=json_type,
prop_value=json.dumps(value))
return value
def __init__(self, action, value):
"""
:param action: one of available action names
:param value: AttributeValue instance, parameter for action
"""
if action not in self._allowed_actions:
raise exception.ValidationError(
_("Update action '%(action)s' isn't allowed"),
action=action
)
super(UpdateItemAction, self).__init__(action=action, value=value)
def parse_local_secondary_indexes(cls, local_secondary_index_list_json):
res = {}
for index_json in local_secondary_index_list_json:
index_name, index_def = (
cls.parse_local_secondary_index(index_json)
)
res[index_name] = index_def
if len(res) < len(local_secondary_index_list_json):
raise exception.ValidationError(
_("Two or more indexes with the same name"))
return res
def validate_unexpected_props(value, property_name):
if len(value) > 0:
if isinstance(value, dict):
value_str = json.dumps(value)
else:
value_str = str(value)
raise exception.ValidationError(
_("Unexpected properties were found for '%(property_name)s': "
"%(unexpected_props)s"),
property_name=property_name,
unexpected_props=value_str
)
return value
def parse_expected_attribute_conditions(
cls, expected_attribute_conditions_json):
expected_attribute_conditions = {}
for (attr_name_json, condition_json) in (
expected_attribute_conditions_json.iteritems()):
validation.validate_attr_name(attr_name_json)
validation.validate_object(condition_json, attr_name_json)
if len(condition_json) != 1:
raise exception.ValidationError(
_("Can't recognize attribute expected condition format: "
"'%(attr)s'"),
attr=json.dumps(condition_json))
(condition_type, condition_value) = condition_json.popitem()
validation.validate_string(condition_type, "condition type")
if condition_type == Props.VALUE:
validation.validate_object(condition_value, Props.VALUE)
expected_attribute_conditions[attr_name_json] = [
models.ExpectedCondition.eq(
cls.parse_typed_attr_value(condition_value))
]
elif condition_type == Props.EXISTS:
validation.validate_boolean(condition_value, Props.EXISTS)
expected_attribute_conditions[attr_name_json] = [
models.ExpectedCondition.not_null()
if condition_value else models.ExpectedCondition.null()
]
else:
raise exception.ValidationError(
_("Unsupported condition type found: %(condition_type)s"),
condition_type=condition_type)
return expected_attribute_conditions
def validate_integer(value, property_name, min_val=None, max_val=None):
if isinstance(value, basestring):
try:
value = int(value)
except ValueError:
pass
value = __validate_type(value, property_name, (int, long), "Integer")
if min_val is not None and value < min_val:
raise exception.ValidationError(_(
"'%(property_name)s' property value[%(property_value)s] is less "
"then min_value[%(min_value)s]."),
property_name=property_name,
property_value=value,
min_value=min_val)
if max_val is not None and value > max_val:
raise exception.ValidationError(_(
"'%(property_name)s' property value[%(property_value)s] is more "
"then max_value[%(max_value)s]."),
property_name=property_name,
property_value=value,
max_value=max_val)
return value
def __validate_type(value, property_name, py_type, json_type):
if value is None:
raise exception.ValidationError(
_("Required property '%(property_name)s' wasn't found "
"or it's value is null"),
property_name=property_name
)
if not isinstance(value, py_type):
raise exception.ValidationError(
WRONG_TYPE_MSG,
property_name=property_name,
json_type=json_type,
prop_value=json.dumps(value)
)
return value
def __init__(self, type, attribute_map):
"""
:param type: one of available type names
:param attribute_map: map of attribute name to AttributeValue instance,
represents item to put or key to delete
"""
if type not in self._allowed_types:
raise exception.ValidationError(
_("Write request_type '%(type)s' isn't allowed"),
type=type
)
super(WriteItemRequest, self).__init__(
type=type, attribute_map=attribute_map
)
def parse_batch_write_request_items(cls, request_items_json):
request_map = {}
for table_name, request_list_json in request_items_json.iteritems():
validation.validate_table_name(table_name)
validation.validate_list_of_objects(request_list_json, table_name)
request_list_for_table = []
for request_json in request_list_json:
for request_type, request_body in request_json.iteritems():
validation.validate_string(request_type, "request_type")
if request_type == Props.REQUEST_PUT:
validation.validate_object(request_body, request_type)
item = request_body.pop(Props.ITEM, None)
validation.validate_object(item, Props.ITEM)
validation.validate_unexpected_props(request_body,
request_type)
request_list_for_table.append(
models.WriteItemRequest.put(
cls.parse_item_attributes(item)
)
)
elif request_type == Props.REQUEST_DELETE:
validation.validate_object(request_body, request_type)
key = request_body.pop(Props.KEY, None)
validation.validate_object(key, Props.KEY)
validation.validate_unexpected_props(request_body,
request_type)
request_list_for_table.append(
models.WriteItemRequest.delete(
cls.parse_item_attributes(key)
)
)
else:
raise exception.ValidationError(
_("Unsupported request type found: "
"%(request_type)s"),
request_type=request_type
)
request_map[table_name] = request_list_for_table
return request_map
def _parse_text_rule(rule):
"""
Translates a policy written in the policy language into a tree of
Check objects.
"""
# Empty rule means always accept
if not rule:
return TrueCheck()
# Parse the token stream
state = ParseState()
for tok, value in _parse_tokenize(rule):
state.shift(tok, value)
try:
return state.result
except ValueError:
# Couldn't parse the rule
LOG.exception(_("Failed to understand rule %s") % rule)
# Fail closed
return FalseCheck()
