def is_dict(value, **kwargs):
"""Indicate whether ``value`` is a valid :class:`dict <python:dict>`
.. note::
This will return ``True`` even if ``value`` is an empty
:class:`dict <python:dict>`.
:param value: The value to evaluate.
:returns: ``True`` if ``value`` is valid, ``False`` if it is not.
:rtype: :class:`bool <python:bool>`
:raises SyntaxError: if ``kwargs`` contains duplicate keyword parameters or duplicates
keyword parameters passed to the underlying validator
"""
if isinstance(value, dict):
return True
try:
value = validators.dict(value, **kwargs)
except SyntaxError as error:
raise error
except Exception:
return False
return True
def value_metadata(self, value):
value = validators.dict(value, allow_empty = True)
if not value:
self._value_metadata = None
else:
self._value_metadata = {
key: validators.string(value[key], allow_empty = True) for key in value
}
def column_metadata(self, value):
value = validators.dict(value, allow_empty = True)
if not value:
self._column_metadata = None
else:
result = {}
for key in value:
key = validators.variable_name(key, allow_empty = False)
if checkers.is_type(value[key], 'ColumnMetadata'):
result[key] = value[key]
else:
result[key] = ColumnMetadata.from_dict(result[key])
self._column_metadata = result
def missing_range_metadata(self, value):
value = validators.iterable(value, allow_empty = True)
if not value:
self._missing_range_metadata = None
else:
ranges = [validators.dict(x, allow_empty = False) for x in value]
validated_ranges = []
for range in ranges:
if 'high' not in range or 'low' not in range:
raise ValueError('missing_range_metadata requires a "high" and "low"'
' boundary to be defined.')
validated_range = {
'high': validators.numeric(range.get('high'), allow_empty = False),
'low': validators.numeric(range.get('low'), allow_empty = False)
}
validated_ranges.append(validated_range)
self._missing_range_metadata = validated_ranges
def is_dict(value, **kwargs):
"""Indicate whether ``value`` is a valid :class:`dict <python:dict>`
.. note::
This will return ``True`` even if ``value`` is an empty
:class:`dict <python:dict>`.
:param value: The value to evaluate.
:returns: ``True`` if ``value`` is valid, ``False`` if it is not.
:rtype: :class:`bool <python:bool>`
"""
if isinstance(value, dict):
return True
try:
value = validators.dict(value, **kwargs)
except Exception:
return False
return True
def from_dict(value):
return validators.dict(value, allow_empty=True, json_serializer=json)
def from_json(value):
value = validators.dict(value, allow_empty=True, json_serializer=json)
if value is None:
return None
return json.dumps(value)
def backoff(to_execute,
args = None,
kwargs = None,
strategy = None,
retry_execute = None,
retry_args = None,
retry_kwargs = None,
max_tries = None,
max_delay = None,
catch_exceptions = None,
on_failure = None,
on_success = None):
"""Retry a function call multiple times with a delay per the strategy given.
:param to_execute: The function call that is to be attempted.
:type to_execute: callable
:param args: The positional arguments to pass to the function on the first attempt.
If ``retry_args`` is :class:`None <python:None>`, will re-use these
arguments on retry attempts as well.
:type args: iterable / :class:`None <python:None>`.
:param kwargs: The keyword arguments to pass to the function on the first attempt.
If ``retry_kwargs`` is :class:`None <python:None>`, will re-use these keyword
arguments on retry attempts as well.
:type kwargs: :class:`dict <python:dict>` / :class:`None <python:None>`
:param strategy: The :class:`BackoffStrategy` to use when determining the
delay between retry attempts.
If :class:`None <python:None>`, defaults to :class:`Exponential`.
:type strategy: :class:`BackoffStrategy`
:param retry_execute: The function to call on retry attempts.
If :class:`None <python:None>`, will retry ``to_execute``.
Defaults to :class:`None <python:None>`.
:type retry_execute: callable / :class:`None <python:None>`
:param retry_args: The positional arguments to pass to the function on retry attempts.
If :class:`None <python:None>`, will re-use ``args``.
Defaults to :class:`None <python:None>`.
:type retry_args: iterable / :class:`None <python:None>`
:param retry_kwargs: The keyword arguments to pass to the function on retry attempts.
If :class:`None <python:None>`, will re-use ``kwargs``.
Defaults to :class:`None <python:None>`.
:type subsequent_kwargs: :class:`dict <python:dict>` / :class:`None <python:None>`
:param max_tries: The maximum number of times to attempt the call.
If :class:`None <python:None>`, will apply an environment variable
``BACKOFF_DEFAULT_TRIES``. If that environment variable is not set, will
apply a default of ``3``.
:type max_tries: int / :class:`None <python:None>`
:param max_delay: The maximum number of seconds to wait befor giving up
once and for all. If :class:`None <python:None>`, will apply an environment variable
``BACKOFF_DEFAULT_DELAY`` if that environment variable is set. If it is not
set, will not apply a max delay at all.
:type max_delay: :class:`None <python:None>` / int
:param catch_exceptions: The ``type(exception)`` to catch and retry. If
:class:`None <python:None>`, will catch all exceptions.
Defaults to :class:`None <python:None>`.
.. caution::
The iterable must contain one or more types of exception *instances*, and not
class objects. For example:
.. code-block:: python
# GOOD:
catch_exceptions = (type(ValueError()), type(TypeError()))
# BAD:
catch_exceptions = (type(ValueError), type(ValueError))
# BAD:
catch_exceptions = (ValueError, TypeError)
# BAD:
catch_exceptions = (ValueError(), TypeError())
:type catch_exceptions: iterable of form ``[type(exception()), ...]``
:param on_failure: The :class:`exception <python:Exception>` or function to call
when all retry attempts have failed.
If :class:`None <python:None>`, will raise the last-caught
:class:`exception <python:Exception>`.
If an :class:`exception <python:Exception>`, will raise the exception with
the same message as the last-caught exception.
If a function, will call the function and pass the last-raised exception, its
message, and stacktrace to the function.
Defaults to :class:`None <python:None>`.
:type on_failure: :class:`Exception <python:Exception>` / function /
:class:`None <python:None>`
:param on_success: The function to call when the operation was successful.
The function receives the result of the ``to_execute`` or ``retry_execute``
function that was successful, and is called before that result is returned
to whatever code called the backoff function. If :class:`None <python:None>`,
will just return the result of ``to_execute`` or ``retry_execute`` without
calling a handler.
Defaults to :class:`None <python:None>`.
:type on_success: callable / :class:`None <python:None>`
:returns: The result of the attempted function.
Example:
.. code-block:: python
from backoff_utils import backoff
def some_function(arg1, arg2, kwarg1 = None):
# Function does something
pass
result = backoff(some_function,
args = ['value1', 'value2'],
kwargs = { 'kwarg1': 'value3' },
max_tries = 3,
max_delay = 30,
strategy = strategies.Exponential)
"""
# pylint: disable=too-many-branches,too-many-statements
if to_execute is None:
raise ValueError('to_execute cannot be None')
elif not checkers.is_callable(to_execute):
raise TypeError('to_execute must be callable')
if strategy is None:
strategy = strategies.Exponential
if not hasattr(strategy, 'IS_INSTANTIATED'):
raise TypeError('strategy must be a BackoffStrategy or descendent')
if not strategy.IS_INSTANTIATED:
test_strategy = strategy(attempt = 0)
else:
test_strategy = strategy
if not checkers.is_type(test_strategy, 'BackoffStrategy'):
raise TypeError('strategy must be a BackoffStrategy or descendent')
if args:
args = validators.iterable(args)
if kwargs:
kwargs = validators.dict(kwargs)
if retry_execute is None:
retry_execute = to_execute
elif not checkers.is_callable(retry_execute):
raise TypeError('retry_execute must be None or a callable')
if not retry_args:
retry_args = args
else:
retry_args = validators.iterable(retry_args)
if not retry_kwargs:
retry_kwargs = kwargs
else:
retry_kwargs = validators.dict(retry_kwargs)
if max_tries is None:
max_tries = DEFAULT_MAX_TRIES
max_tries = validators.integer(max_tries)
if max_delay is None:
max_delay = DEFAULT_MAX_DELAY
if catch_exceptions is None:
catch_exceptions = [type(Exception())]
else:
if not checkers.is_iterable(catch_exceptions):
catch_exceptions = [catch_exceptions]
catch_exceptions = validators.iterable(catch_exceptions)
if on_failure is not None and not checkers.is_callable(on_failure):
raise TypeError('on_failure must be None or a callable')
if on_success is not None and not checkers.is_callable(on_success):
raise TypeError('on_success must be None or a callable')
cached_error = None
return_value = None
returned = False
failover_counter = 0
start_time = datetime.utcnow()
while failover_counter <= (max_tries):
elapsed_time = (datetime.utcnow() - start_time).total_seconds()
if max_delay is not None and elapsed_time >= max_delay:
if cached_error is None:
raise BackoffTimeoutError('backoff timed out after:'
' {}s'.format(elapsed_time))
else:
_handle_failure(on_failure, cached_error)
if failover_counter == 0:
try:
if args is not None and kwargs is not None:
return_value = to_execute(*args, **kwargs)
elif args is not None:
return_value = to_execute(*args)
elif kwargs is not None:
return_value = to_execute(**kwargs)
else:
return_value = to_execute()
returned = True
break
except Exception as error: # pylint: disable=broad-except
if type(error) in catch_exceptions:
cached_error = error
strategy.delay(failover_counter)
failover_counter += 1
continue
else:
_handle_failure(on_failure = on_failure,
error = error)
return
else:
try:
if retry_args is not None and retry_kwargs is not None:
return_value = retry_execute(*retry_args, **retry_kwargs)
elif retry_args is not None:
return_value = retry_execute(*retry_args)
elif retry_kwargs is not None:
return_value = retry_execute(**retry_kwargs)
else:
return_value = retry_execute()
returned = True
break
except Exception as error: # pylint: disable=broad-except
if type(error) in catch_exceptions:
strategy.delay(failover_counter)
cached_error = error
failover_counter += 1
continue
else:
_handle_failure(on_failure = on_failure,
error = error)
return
if not returned:
_handle_failure(on_failure = on_failure,
error = cached_error)
return
elif returned and on_success is not None:
on_success(return_value)
return return_value
def from_dict(cls, obj, api_compatible=False):
"""Create a :class:`LocationScore` instance from a
:class:`dict <python:dict>` representation.
:param obj: The :class:`dict <python:dict>` representation of the location
score.
:type obj: :class:`dict <python:dict>`
:param api_compatible: If ``True``, expects ``obj`` to be a
:class:`dict <python:dict>` whose structure is compatible with the
JSON object returned by the WalkScore API. If ``False``, expects a
slightly more normalized :class:`dict <python:dict>` representation.
Defaults to ``False``.
:type api_compatible: :class:`bool <python:bool>`
:returns: :class:`LocationScore` representation of ``obj``.
:rtype: :class:`LocationScore`
"""
obj = validators.dict(obj, allow_empty=True)
result = cls()
if obj and not api_compatible:
result.address = obj.get('original_coordinates',
{}).get('address', None)
result.original_latitude = obj.get('original_coordinates',
{}).get('latitude')
result.original_longitude = obj.get('original_coordinates',
{}).get('longitude')
result.snapped_latitude = obj.get('snapped_coordinates',
{}).get('latitude')
result.snapped_longitude = obj.get('snapped_coordinates',
{}).get('longitude')
result.walk_score = obj.get('walk', {}).get('score', None)
result.walk_description = obj.get('walk',
{}).get('description', None)
result.walk_updated = obj.get('walk', {}).get('updated', None)
result.property_page_link = obj.get('property_page_link', None)
elif obj:
result.walk_score = obj.get('walkscore', None)
result.walk_description = obj.get('description', None)
result.walk_updated = obj.get('updated', None)
result.snapped_latitude = obj.get('snapped_lat', None)
result.snapped_longitude = obj.get('snapped_lon', None)
result.property_page_link = obj.get('ws_link', None)
if obj:
result.status = obj.get('status', None)
result.transit_score = obj.get('transit', {}).get('score', None)
result.transit_description = obj.get('transit',
{}).get('description', None)
result.transit_summary = obj.get('transit',
{}).get('summary', None)
result.bike_score = obj.get('bike', {}).get('score', None)
result.bike_description = obj.get('bike',
{}).get('description', None)
result.logo_url = obj.get('logo_url', None)
result.more_info_icon = obj.get('more_info_icon', None)
result.more_info_link = obj.get('more_info_link', None)
result.help_link = obj.get('help_link', None)
return result
def request(self,
method,
url,
parameters=None,
headers=None,
request_body=None):
"""Execute a standard HTTP request.
:param method: The HTTP method to use for the request. Accepts `GET`, `HEAD`,
`POST`, `PATCH`, `PUT`, or `DELETE`.
:type method: :class:`str <python:str>`
:param url: The URL to execute the request against.
:type url: :class:`str <python:str>`
:param parameters: URL parameters to submit with the request. Defaults to
:obj:`None <python:None>`.
:type parameters: :class:`dict <python:dict>` / :obj:`None <python:None>`
:param headers: HTTP headers to submit with the request. Defaults to
:obj:`None <python:None>`.
:type headers: :class:`dict <python:dict>` / :obj:`None <python:None>`
:param request_body: The data to supply in the body of the request. Defaults to
:obj:`None <python:None>`.
:type request_body: :obj:`None <python:None>` / :class:`dict <python:dict>` /
:class:`str <python:str>` / :class:`bytes <python:bytes>`
:returns: The content of the HTTP response, the status code of the HTTP response,
and the headers of the HTTP response.
:rtype: :class:`tuple <python:tuple>` of :class:`bytes <python:bytes>`,
:class:`int <python:int>`, and :class:`dict <python:dict>`
:raises ValueError: if ``method`` is not either ``GET``, ``HEAD``, ``POST``,
``PATCH``, ``PUT`` or ``DELETE``
:raises ValueError: if ``url`` is not a valid URL
:raises ValueError: if ``headers`` is not empty and is not a
:class:`dict <python:dict>`
:raises HTTPTimeoutError: if the request times out
:raises SSLError: if the request fails SSL certificate verification
:raises WalkScoreError: *or sub-classes* for other errors returned by the API
"""
method = validators.string(method, allow_empty=False)
method = method.upper()
if method not in HTTP_METHODS:
raise ValueError('method (%s) not a recognized HTTP method' %
method)
url = validators.url(url, allow_empty=False)
parameters = validators.dict(parameters, allow_empty=True)
headers = validators.dict(headers, allow_empty=True)
content, status_code, headers = self._request(method, url, parameters,
headers, request_body)
check_for_errors(status_code, content)
return content, status_code, headers
def _parse_dict(cls,
input_data,
format,
error_on_extra_keys=True,
drop_extra_keys=False,
config_set=None):
"""Generate a processed :class:`dict <python:dict>` object from
in-bound :class:`dict <python:dict>` data.
:param input_data: An inbound :class:`dict <python:dict>` object which
can be processed for de-serialization.
:type input_data: :class:`dict <python:dict>`
:param format: The format from which ``input_data`` was received. Accepts:
``'csv'``, ``'json'``, ``'yaml'``, and ``'dict'``.
:type format: :class:`str <python:str>`
:param error_on_extra_keys: If ``True``, will raise an error if an
unrecognized key is found in ``dict_data``. If ``False``, will
either drop or include the extra key in the result, as configured in
the ``drop_extra_keys`` parameter. Defaults to ``True``.
:type error_on_extra_keys: :class:`bool <python:bool>`
:param drop_extra_keys: If ``True``, will omit unrecognized top-level keys
from the resulting :class:`dict <python:dict>`. If ``False``, will
include unrecognized keys or raise an error based on the configuration of
the ``error_on_extra_keys`` parameter. Defaults to ``False``.
:type drop_extra_keys: :class:`bool <python:bool>`
:param config_set: If not :obj:`None <python:None>`, the named configuration set
to use when processing the input. Defaults to :obj:`None <python:None>`.
:type config_set: :class:`str <python:str>` / :obj:`None <python:None>`
:returns: A processed :class:`dict <python:dict>` object that has had
:term:`deserializer functions` applied to it.
:rtype: :class:`dict <python:dict>`
:raises ExtraKeyError: if ``error_on_extra_keys`` is ``True`` and
``input_data`` contains top-level keys that are not recognized as
attributes for the instance model.
:raises DeserializationError: if ``input_data`` is
not a :class:`dict <python:dict>` or JSON object serializable to a
:class:`dict <python:dict>` or if ``input_data`` is empty.
:raises InvalidFormatError: if ``format`` is not a supported value
"""
if format not in ['csv', 'json', 'yaml', 'dict']:
raise InvalidFormatError("format '%s' not supported" % format)
try:
input_data = validators.dict(input_data,
allow_empty=True,
json_serializer=json)
except ValueError:
raise DeserializationError('input_data is not a dict')
if not input_data or len(input_data.keys()) == 0:
raise DeserializationError("input_data is empty")
dict_object = dict_()
if format == 'csv':
attribute_getter = cls.get_csv_serialization_config
elif format == 'json':
attribute_getter = cls.get_json_serialization_config
elif format == 'yaml':
attribute_getter = cls.get_yaml_serialization_config
elif format == 'dict':
attribute_getter = cls.get_dict_serialization_config
attributes = [
x for x in attribute_getter(
deserialize=True, serialize=None, config_set=config_set)
if hasattr(cls, x.name)
]
if not attributes:
raise DeserializableAttributeError(
"'%s' has no '%s' de-serializable attributes" %
(type(cls.__name__), format))
attribute_names = [x.display_name or x.name for x in attributes]
extra_keys = [x for x in input_data.keys() if x not in attribute_names]
if extra_keys and error_on_extra_keys:
raise ExtraKeyError("input data had extra keys: %s" % extra_keys)
for attribute in attributes:
key = attribute.display_name or attribute.name
try:
value = input_data.pop(key)
except KeyError:
continue
value = cls._get_deserialized_value(
value,
format,
key,
error_on_extra_keys=error_on_extra_keys,
drop_extra_keys=drop_extra_keys,
config_set=config_set)
dict_object[attribute.name] = value
if input_data and not drop_extra_keys:
for key in input_data:
dict_object[key] = input_data[key]
return dict_object