def parse_json(input_data, deserialize_function=None, **kwargs):
"""De-serialize JSON data into a Python :class:`dict <python:dict>` object.
:param input_data: The JSON data to de-serialize.
:type input_data: :class:`str <python:str>`
:param deserialize_function: Optionally override the default JSON deserializer.
Defaults to :obj:`None <python:None>`, which calls the default
:ref:`simplejson.loads() <simplejson:simplejson.loads>`
function from the `simplejson <https://github.com/simplejson/simplejson>`_ library.
.. note::
Use the ``deserialize_function`` parameter to override the default
YAML deserializer. A valid ``deserialize_function`` is expected to
accept a single :class:`str <python:str>` and return a
:class:`dict <python:dict>`, similar to
:ref:`simplejson.loads() <simplejson:simplejson.loads>`
If you wish to pass additional arguments to your ``deserialize_function``
pass them as keyword arguments (in ``kwargs``).
:type deserialize_function: callable / :obj:`None <python:None>`
:param kwargs: Optional keyword parameters that are passed to the
JSON deserializer function. By default, these are options which are passed
to :ref:`simplejson.loads() <simplejson:simplejson.loads>`.
:type kwargs: keyword arguments
:returns: A :class:`dict <python:dict>` representation of ``input_data``.
:rtype: :class:`dict <python:dict>`
"""
is_file = False
if checkers.is_file(input_data):
is_file = True
if deserialize_function is None and not is_file:
deserialize_function = json.loads
elif deserialize_function is None and is_file:
deserialize_function = json.load
else:
if checkers.is_callable(deserialize_function) is False:
raise ValueError('deserialize_function (%s) is not callable' %
deserialize_function)
if not input_data:
raise DeserializationError('input_data is empty')
if not is_file:
try:
input_data = validators.string(input_data, allow_empty=False)
except ValueError:
raise DeserializationError('input_data is not a valid string')
from_json = deserialize_function(input_data, **kwargs)
else:
with open(input_data, 'r') as input_file:
from_json = deserialize_function(input_file, **kwargs)
return from_json
def on_deserialize(self, value):
value = callable_to_dict(value)
for key in value:
item = value[key]
if item is not None and not checkers.is_callable(item):
raise SQLAthanorError('on_deserialize for %s must be callable' % key)
self._on_deserialize = value
def parse_yaml(input_data, deserialize_function=None, **kwargs):
"""De-serialize YAML data into a Python :class:`dict <python:dict>` object.
:param input_data: The YAML data to de-serialize.
:type input_data: :class:`str <python:str>`
:param deserialize_function: Optionally override the default YAML deserializer.
Defaults to :obj:`None <python:None>`, which calls the default ``yaml.safe_load()``
function from the `PyYAML <https://github.com/yaml/pyyaml>`_ library.
.. note::
Use the ``deserialize_function`` parameter to override the default
YAML deserializer. A valid ``deserialize_function`` is expected to
accept a single :class:`str <python:str>` and return a
:class:`dict <python:dict>`, similar to ``yaml.safe_load()``.
If you wish to pass additional arguments to your ``deserialize_function``
pass them as keyword arguments (in ``kwargs``).
:type deserialize_function: callable / :obj:`None <python:None>`
:param kwargs: Optional keyword parameters that are passed to the
YAML deserializer function. By default, these are options which are passed
to ``yaml.safe_load()``.
:type kwargs: keyword arguments
:returns: A :class:`dict <python:dict>` representation of ``input_data``.
:rtype: :class:`dict <python:dict>`
"""
if deserialize_function is None:
deserialize_function = yaml.safe_load
else:
if checkers.is_callable(deserialize_function) is False:
raise ValueError('deserialize_function (%s) is not callable' %
deserialize_function)
if not input_data:
raise DeserializationError('input_data is empty')
try:
input_data = validators.string(input_data, allow_empty=False)
except ValueError:
raise DeserializationError('input_data is not a valid string')
from_yaml = yaml.safe_load(input_data, **kwargs)
return from_yaml
def __init__(self,
argument,
supports_json=False,
supports_yaml=False,
supports_dict=False,
on_serialize=None,
on_deserialize=None,
**kwargs):
"""Provide a relationship between two mapped classes.
This corresponds to a parent-child or associate table relationship.
The constructed class is an instance of :class:`RelationshipProperty`.
When serializing or de-serializing relationships, they essentially become
"nested" objects. For example, if you have an ``Account`` table with a
relationship to a ``User`` table, you might want to nest or embed a list of
``User`` objects within a serialized ``Account`` object.
.. caution::
Unlike columns, properties, or hybrid properties, relationships
cannot be serialized to CSV. This is because a serialized relationship
is essentially a "nested" object within another object.
Therefore, the ``supports_csv`` option cannot be set and will always be
interpreted as ``False``.
.. warning::
This constructor is analogous to the original
:ref:`SQLAlchemy relationship() <sqlalchemy:sqlalchemy.orm.relationship>`
from which it inherits. The only difference is that it supports additional
keyword arguments which are not supported in the original, and which
are documented below.
**For the original SQLAlchemy version, see:**
:ref:`(SQLAlchemy) relationship() <sqlalchemy:sqlalchemy.orm.relationship>`
:param argument: see
:ref:`(SQLAlchemy) relationship() <sqlalchemy:sqlalchemy.orm.relationship>`
:param supports_json: Determines whether the column can be serialized to or
de-serialized from JSON format. If ``True``, can be serialized to JSON and
de-serialized from JSON. If ``False``, will not be included when serialized
to JSON and will be ignored if present in a de-serialized JSON.
Can also accept a 2-member :class:`tuple <python:tuple>` (inbound / outbound)
which determines de-serialization and serialization support respectively.
Defaults to ``False``, which means the column will not be serialized to JSON
or de-serialized from JSON.
:type supports_json: :class:`bool <python:bool>` / :class:`tuple <python:tuple>` of
form (inbound: :class:`bool <python:bool>`, outbound: :class:`bool <python:bool>`)
:param supports_yaml: Determines whether the column can be serialized to or
de-serialized from YAML format. If ``True``, can be serialized to YAML and
de-serialized from YAML. If ``False``, will not be included when serialized
to YAML and will be ignored if present in a de-serialized YAML.
Can also accept a 2-member :class:`tuple <python:tuple>` (inbound / outbound)
which determines de-serialization and serialization support respectively.
Defaults to ``False``, which means the column will not be serialized to YAML
or de-serialized from YAML.
:type supports_yaml: :class:`bool <python:bool>` / :class:`tuple <python:tuple>` of
form (inbound: :class:`bool <python:bool>`, outbound: :class:`bool <python:bool>`)
:param supports_dict: Determines whether the column can be serialized to or
de-serialized to a Python :class:`dict <python:dict>`. If ``True``, can
be serialized to :class:`dict <python:dict>` and de-serialized from a
:class:`dict <python:dict>`. If ``False``, will not be included when serialized
to :class:`dict <python:dict>` and will be ignored if present in a de-serialized
:class:`dict <python:dict>`.
Can also accept a 2-member :class:`tuple <python:tuple>` (inbound / outbound)
which determines de-serialization and serialization support respectively.
Defaults to ``False``, which means the column will not be serialized to a
:class:`dict <python:dict>` or de-serialized from a :class:`dict <python:dict>`.
:type supports_dict: :class:`bool <python:bool>` / :class:`tuple <python:tuple>` of
form (inbound: :class:`bool <python:bool>`, outbound: :class:`bool <python:bool>`)
"""
# pylint: disable=too-many-branches
if on_serialize is not None and not isinstance(on_serialize, dict):
on_serialize = {
'csv': on_serialize,
'json': on_serialize,
'yaml': on_serialize,
'dict': on_serialize
}
elif on_serialize is not None:
if 'csv' not in on_serialize:
on_serialize['csv'] = None
if 'json' not in on_serialize:
on_serialize['json'] = None
if 'yaml' not in on_serialize:
on_serialize['yaml'] = None
if 'dict' not in on_serialize:
on_serialize['dict'] = None
else:
on_serialize = {
'csv': None,
'json': None,
'yaml': None,
'dict': None
}
for key in on_serialize:
item = on_serialize[key]
if item is not None and not checkers.is_callable(item):
raise SQLAthanorError('on_serialize for %s must be callable' %
key)
if on_deserialize is not None and not isinstance(on_deserialize, dict):
on_deserialize = {
'csv': on_deserialize,
'json': on_deserialize,
'yaml': on_deserialize,
'dict': on_deserialize
}
elif on_deserialize is not None:
if 'csv' not in on_deserialize:
on_deserialize['csv'] = None
if 'json' not in on_deserialize:
on_deserialize['json'] = None
if 'yaml' not in on_deserialize:
on_deserialize['yaml'] = None
if 'dict' not in on_deserialize:
on_deserialize['dict'] = None
else:
on_deserialize = {
'csv': None,
'json': None,
'yaml': None,
'dict': None
}
for key in on_deserialize:
item = on_deserialize[key]
if item is not None and not checkers.is_callable(item):
raise SQLAthanorError(
'on_deserialize for %s must be callable' % key)
if supports_json is True:
supports_json = (True, True)
elif not supports_json:
supports_json = (False, False)
if supports_yaml is True:
supports_yaml = (True, True)
elif not supports_yaml:
supports_yaml = (False, False)
if supports_dict is True:
supports_dict = (True, True)
elif not supports_dict:
supports_dict = (False, False)
self.supports_csv = (False, False)
self.csv_sequence = None
self.supports_json = supports_json
self.supports_yaml = supports_yaml
self.supports_dict = supports_dict
self.on_serialize = on_serialize
self.on_deserialize = on_deserialize
comparator_factory = kwargs.pop('comparator_factory',
RelationshipProperty.Comparator)
super(RelationshipProperty,
self).__init__(argument,
comparator_factory=comparator_factory,
**kwargs)
def get_attribute_names(obj,
include_callable=False,
include_nested=True,
include_private=False,
include_special=False,
include_utilities=False):
"""Return a list of attribute names within ``obj``.
:param include_callable: If ``True``, will include callable attributes (methods).
Defaults to ``False``.
:type include_callable: :class:`bool <python:bool>`
:param include_nested: If ``True``, will include attributes that are
arbitrarily-nestable types (such as a :term:`model class` or
:class:`dict <python:dict>`). Defaults to ``False``.
:type include_nested: :class:`bool <python:bool>`
:param include_private: If ``True``, will include attributes whose names
begin with ``_`` (but *not* ``__``). Defaults to ``False``.
:type include_private: :class:`bool <python:bool>`
:param include_special: If ``True``, will include atributes whose names begin
with ``__``. Defaults to ``False``.
:type include_special: :class:`bool <python:bool>`
:param include_utilities: If ``True``, will include utility properties
added by SQLAlchemy or **SQLAthanor**. Defaults to ``False``.
:type include_utilities: :class:`bool <python:bool>`
:returns: :term:`Model Attribute` names attached to ``obj``.
:rtype: :class:`list <python:list>` of :class:`str <python:str>`
"""
attribute_names = [x for x in dir(obj)
if (include_utilities and x in UTILITY_COLUMNS) or \
(x not in UTILITY_COLUMNS)]
attributes = []
for attribute in attribute_names:
if (attribute[0] == '_'
and attribute[0:2] != '__') and not include_private:
continue
if attribute[0:2] == '__' and not include_special:
continue
try:
attribute_value = getattr(obj, attribute)
except SA_InvalidRequestError:
if not include_nested:
continue
attributes.append(attribute)
continue
if not include_nested:
if checkers.is_type(attribute_value,
('BaseModel', 'RelationshipProperty',
'AssociationProxy', dict)):
continue
try:
is_iterable = checkers.is_iterable(attribute_value,
forbid_literals=(str, bytes,
dict))
except SA_InvalidRequestError as error:
if not include_nested:
continue
else:
is_iterable = False
if is_iterable:
loop = False
try:
for item in attribute_value:
if checkers.is_type(
item, ('BaseModel', 'RelationshipProperty',
'AssociationProxy', dict)):
loop = True
break
except (NotImplementedError, TypeError):
pass
if loop:
continue
if not include_callable and checkers.is_callable(attribute_value):
continue
attributes.append(attribute)
return attributes
def dump_to_json(self,
max_nesting=0,
current_nesting=0,
serialize_function=None,
config_set=None,
**kwargs):
"""Return a :term:`JSON <JavaScript Object Notation (JSON)>`
representation of the object, *with all attributes* regardless of
configuration.
.. caution::
Nested objects (such as :term:`relationships <relationship>` or
:term:`association proxies <association proxy>`) will **not**
be serialized.
:param max_nesting: The maximum number of levels that the resulting
JSON object can be nested. If set to ``0``, will
not nest other serializable objects. Defaults to ``0``.
:type max_nesting: :class:`int <python:int>`
:param current_nesting: The current nesting level at which the
:class:`dict <python:dict>` representation will reside. Defaults to ``0``.
:type current_nesting: :class:`int <python:int>`
:param serialize_function: Optionally override the default JSON serializer.
Defaults to :obj:`None <python:None>`, which applies the default
:doc:`simplejson <simplejson:index>` JSON serializer.
.. note::
Use the ``serialize_function`` parameter to override the default
JSON serializer.
A valid ``serialize_function`` is expected to accept a single
:class:`dict <python:dict>` and return a :class:`str <python:str>`,
similar to :func:`simplejson.dumps() <simplejson:simplejson.dumps>`.
If you wish to pass additional arguments to your ``serialize_function``
pass them as keyword arguments (in ``kwargs``).
:type serialize_function: callable / :obj:`None <python:None>`
:param config_set: If not :obj:`None <python:None>`, the named configuration set
to use. Defaults to :obj:`None <python:None>`.
:type config_set: :class:`str <python:str>` / :obj:`None <python:None>`
:param kwargs: Optional keyword parameters that are passed to the
JSON serializer function. By default, these are options which are passed
to :func:`simplejson.dumps() <simplejson:simplejson.dumps>`.
:type kwargs: keyword arguments
:returns: A :class:`str <python:str>` with the JSON representation of the
object.
:rtype: :class:`str <python:str>`
:raises SerializableAttributeError: if attributes is empty
:raises MaximumNestingExceededError: if ``current_nesting`` is greater
than ``max_nesting``
:raises MaximumNestingExceededWarning: if an attribute requires nesting
beyond ``max_nesting``
"""
if serialize_function is None:
serialize_function = json.dumps
else:
if checkers.is_callable(serialize_function) is False:
raise ValueError('serialize_function (%s) is not callable' %
serialize_function)
as_dict = self._to_dict('json',
max_nesting=max_nesting,
current_nesting=current_nesting,
is_dumping=True,
config_set=config_set)
as_json = serialize_function(as_dict, **kwargs)
return as_json
def dump_to_yaml(self,
max_nesting = 0,
current_nesting = 0,
serialize_function = None,
config_set = None,
**kwargs):
"""Return a :term:`YAML <YAML Ain't a Markup Language (YAML)>`
representation of the object *with all attributes*, regardless of
configuration.
.. caution::
Nested objects (such as :term:`relationships <relationship>` or
:term:`association proxies <association proxy>`) will **not**
be serialized.
:param max_nesting: The maximum number of levels that the resulting
object can be nested. If set to ``0``, will not nest other serializable
objects. Defaults to ``0``.
:type max_nesting: :class:`int <python:int>`
:param current_nesting: The current nesting level at which the
representation will reside. Defaults to ``0``.
:type current_nesting: :class:`int <python:int>`
:param serialize_function: Optionally override the default YAML serializer.
Defaults to :obj:`None <python:None>`, which calls the default ``yaml.dump()``
function from the `PyYAML <https://github.com/yaml/pyyaml>`_ library.
.. note::
Use the ``serialize_function`` parameter to override the default
YAML serializer.
A valid ``serialize_function`` is expected to
accept a single :class:`dict <python:dict>` and return a
:class:`str <python:str>`, similar to ``yaml.dump()``.
If you wish to pass additional arguments to your ``serialize_function``
pass them as keyword arguments (in ``kwargs``).
:type serialize_function: callable / :obj:`None <python:None>`
:param config_set: If not :obj:`None <python:None>`, the named configuration set
to use. Defaults to :obj:`None <python:None>`.
:type config_set: :class:`str <python:str>` / :obj:`None <python:None>`
:param kwargs: Optional keyword parameters that are passed to the
YAML serializer function. By default, these are options which are passed
to ``yaml.dump()``.
:type kwargs: keyword arguments
:returns: A :class:`str <python:str>` with the JSON representation of the
object.
:rtype: :class:`str <python:str>`
:raises SerializableAttributeError: if attributes is empty
:raises MaximumNestingExceededError: if ``current_nesting`` is greater
than ``max_nesting``
:raises MaximumNestingExceededWarning: if an attribute requires nesting
beyond ``max_nesting``
"""
if serialize_function is None:
serialize_function = yaml.dump
else:
if checkers.is_callable(serialize_function) is False:
raise ValueError(
'serialize_function (%s) is not callable' % serialize_function
)
as_dict = self._to_dict('yaml',
max_nesting = max_nesting,
current_nesting = current_nesting,
is_dumping = True,
config_set = config_set)
if isinstance(as_dict, OrderedDict):
as_dict = dict(as_dict)
as_yaml = serialize_function(as_dict,
**kwargs)
return as_yaml
def get_type_mapping(value,
type_mapping=None,
skip_nested=True,
default_to_str=False):
"""Retrieve the SQL type mapping for ``value``.
:param value: The value whose SQL type will be returned.
:param type_mapping: Determines how the value type of ``value`` map to
SQL column data types. To add a new mapping or override a default, set a
key to the name of the value type in Python, and set the value to a
:doc:`SQLAlchemy Data Type <sqlalchemy:core/types>`. The following are the
default mappings applied:
.. list-table::
:widths: 30 30
:header-rows: 1
* - Python Literal
- SQL Column Type
* - ``bool``
- :class:`Boolean <sqlalchemy:sqlalchemy.types.Boolean>`
* - ``str``
- :class:`Text <sqlalchemy:sqlalchemy.types.Text>`
* - ``int``
- :class:`Integer <sqlalchemy:sqlalchemy.types.Integer>`
* - ``float``
- :class:`Float <sqlalchemy:sqlalchemy.types.Float>`
* - ``date``
- :class:`Date <sqlalchemy:sqlalchemy.types.Date>`
* - ``datetime``
- :class:`DateTime <sqlalchemy:sqlalchemy.types.DateTime>`
* - ``time``
- :class:`Time <sqlalchemy:sqlalchemy.types.Time>`
* - ``timedelta``
- :class:`Interval <sqlalchemy:sqlalchemy.types.Interval>`
:type type_mapping: :class:`dict <python:dict>` with type names as keys and
column data types as values / :obj:`None <python:None>`
:param skip_nested: If ``True`` then if ``value`` is a nested item (e.g.
iterable, :class:`dict <python:dict>` objects, etc.) it will return
:obj:`None <python:None>`. If ``False``, will treat nested items as
:class:`str <python:str>`. Defaults to ``True``.
:type skip_nested: :class:`bool <python:bool>`
:param default_to_str: If ``True``, will automatically set a ``value`` whose
value type cannot be determined to ``str``
(:class:`Text <sqlalchemy:sqlalchemy.types.Text>`). If ``False``, will
use the value type's ``__name__`` attribute and attempt to find a mapping.
Defaults to ``False``.
:type default_to_str: :class:`bool <python:bool>`
:returns: The :doc:`SQLAlchemy Data Type <sqlalchemy:core/types>` for ``value``, or
:obj:`None <python:None>` if the value should be skipped
:rtype: :doc:`SQLAlchemy Data Type <sqlalchemy:core/types>` / :obj:`None`
:raises UnsupportedValueTypeError: when ``value`` does not have corresponding
:doc:`SQLAlchemy Data Type <sqlalchemy:core/types>`
"""
if not type_mapping:
type_mapping = DEFAULT_PYTHON_SQL_TYPE_MAPPING
for key in DEFAULT_PYTHON_SQL_TYPE_MAPPING:
if key not in type_mapping:
type_mapping[key] = DEFAULT_PYTHON_SQL_TYPE_MAPPING[key]
if checkers.is_callable(value):
raise UnsupportedValueTypeError('value ("%s") cannot be callable' %
value)
elif checkers.is_iterable(value) and skip_nested:
return None
elif checkers.is_iterable(value) and default_to_str:
target_type = 'str'
elif value is None and default_to_str:
target_type = 'str'
elif isinstance(value, bool):
target_type = 'bool'
elif checkers.is_numeric(value):
if checkers.is_integer(value):
target_type = 'int'
else:
target_type = 'float'
elif checkers.is_time(value) and not checkers.is_datetime(value):
target_type = 'time'
elif checkers.is_datetime(value):
target_type = 'datetime'
elif checkers.is_date(value):
target_type = 'date'
elif default_to_str:
target_type = 'str'
else:
target_type = type(value).__name__
column_type = type_mapping.get(target_type, None)
if not column_type:
raise UnsupportedValueTypeError(
'value ("%s") is not a supported type (%s)' % (value, target_type))
return column_type
def backoff(to_execute,
args = None,
kwargs = None,
strategy = None,
retry_execute = None,
retry_args = None,
retry_kwargs = None,
max_tries = None,
max_delay = None,
catch_exceptions = None,
on_failure = None,
on_success = None):
"""Retry a function call multiple times with a delay per the strategy given.
:param to_execute: The function call that is to be attempted.
:type to_execute: callable
:param args: The positional arguments to pass to the function on the first attempt.
If ``retry_args`` is :class:`None <python:None>`, will re-use these
arguments on retry attempts as well.
:type args: iterable / :class:`None <python:None>`.
:param kwargs: The keyword arguments to pass to the function on the first attempt.
If ``retry_kwargs`` is :class:`None <python:None>`, will re-use these keyword
arguments on retry attempts as well.
:type kwargs: :class:`dict <python:dict>` / :class:`None <python:None>`
:param strategy: The :class:`BackoffStrategy` to use when determining the
delay between retry attempts.
If :class:`None <python:None>`, defaults to :class:`Exponential`.
:type strategy: :class:`BackoffStrategy`
:param retry_execute: The function to call on retry attempts.
If :class:`None <python:None>`, will retry ``to_execute``.
Defaults to :class:`None <python:None>`.
:type retry_execute: callable / :class:`None <python:None>`
:param retry_args: The positional arguments to pass to the function on retry attempts.
If :class:`None <python:None>`, will re-use ``args``.
Defaults to :class:`None <python:None>`.
:type retry_args: iterable / :class:`None <python:None>`
:param retry_kwargs: The keyword arguments to pass to the function on retry attempts.
If :class:`None <python:None>`, will re-use ``kwargs``.
Defaults to :class:`None <python:None>`.
:type subsequent_kwargs: :class:`dict <python:dict>` / :class:`None <python:None>`
:param max_tries: The maximum number of times to attempt the call.
If :class:`None <python:None>`, will apply an environment variable
``BACKOFF_DEFAULT_TRIES``. If that environment variable is not set, will
apply a default of ``3``.
:type max_tries: int / :class:`None <python:None>`
:param max_delay: The maximum number of seconds to wait befor giving up
once and for all. If :class:`None <python:None>`, will apply an environment variable
``BACKOFF_DEFAULT_DELAY`` if that environment variable is set. If it is not
set, will not apply a max delay at all.
:type max_delay: :class:`None <python:None>` / int
:param catch_exceptions: The ``type(exception)`` to catch and retry. If
:class:`None <python:None>`, will catch all exceptions.
Defaults to :class:`None <python:None>`.
.. caution::
The iterable must contain one or more types of exception *instances*, and not
class objects. For example:
.. code-block:: python
# GOOD:
catch_exceptions = (type(ValueError()), type(TypeError()))
# BAD:
catch_exceptions = (type(ValueError), type(ValueError))
# BAD:
catch_exceptions = (ValueError, TypeError)
# BAD:
catch_exceptions = (ValueError(), TypeError())
:type catch_exceptions: iterable of form ``[type(exception()), ...]``
:param on_failure: The :class:`exception <python:Exception>` or function to call
when all retry attempts have failed.
If :class:`None <python:None>`, will raise the last-caught
:class:`exception <python:Exception>`.
If an :class:`exception <python:Exception>`, will raise the exception with
the same message as the last-caught exception.
If a function, will call the function and pass the last-raised exception, its
message, and stacktrace to the function.
Defaults to :class:`None <python:None>`.
:type on_failure: :class:`Exception <python:Exception>` / function /
:class:`None <python:None>`
:param on_success: The function to call when the operation was successful.
The function receives the result of the ``to_execute`` or ``retry_execute``
function that was successful, and is called before that result is returned
to whatever code called the backoff function. If :class:`None <python:None>`,
will just return the result of ``to_execute`` or ``retry_execute`` without
calling a handler.
Defaults to :class:`None <python:None>`.
:type on_success: callable / :class:`None <python:None>`
:returns: The result of the attempted function.
Example:
.. code-block:: python
from backoff_utils import backoff
def some_function(arg1, arg2, kwarg1 = None):
# Function does something
pass
result = backoff(some_function,
args = ['value1', 'value2'],
kwargs = { 'kwarg1': 'value3' },
max_tries = 3,
max_delay = 30,
strategy = strategies.Exponential)
"""
# pylint: disable=too-many-branches,too-many-statements
if to_execute is None:
raise ValueError('to_execute cannot be None')
elif not checkers.is_callable(to_execute):
raise TypeError('to_execute must be callable')
if strategy is None:
strategy = strategies.Exponential
if not hasattr(strategy, 'IS_INSTANTIATED'):
raise TypeError('strategy must be a BackoffStrategy or descendent')
if not strategy.IS_INSTANTIATED:
test_strategy = strategy(attempt = 0)
else:
test_strategy = strategy
if not checkers.is_type(test_strategy, 'BackoffStrategy'):
raise TypeError('strategy must be a BackoffStrategy or descendent')
if args:
args = validators.iterable(args)
if kwargs:
kwargs = validators.dict(kwargs)
if retry_execute is None:
retry_execute = to_execute
elif not checkers.is_callable(retry_execute):
raise TypeError('retry_execute must be None or a callable')
if not retry_args:
retry_args = args
else:
retry_args = validators.iterable(retry_args)
if not retry_kwargs:
retry_kwargs = kwargs
else:
retry_kwargs = validators.dict(retry_kwargs)
if max_tries is None:
max_tries = DEFAULT_MAX_TRIES
max_tries = validators.integer(max_tries)
if max_delay is None:
max_delay = DEFAULT_MAX_DELAY
if catch_exceptions is None:
catch_exceptions = [type(Exception())]
else:
if not checkers.is_iterable(catch_exceptions):
catch_exceptions = [catch_exceptions]
catch_exceptions = validators.iterable(catch_exceptions)
if on_failure is not None and not checkers.is_callable(on_failure):
raise TypeError('on_failure must be None or a callable')
if on_success is not None and not checkers.is_callable(on_success):
raise TypeError('on_success must be None or a callable')
cached_error = None
return_value = None
returned = False
failover_counter = 0
start_time = datetime.utcnow()
while failover_counter <= (max_tries):
elapsed_time = (datetime.utcnow() - start_time).total_seconds()
if max_delay is not None and elapsed_time >= max_delay:
if cached_error is None:
raise BackoffTimeoutError('backoff timed out after:'
' {}s'.format(elapsed_time))
else:
_handle_failure(on_failure, cached_error)
if failover_counter == 0:
try:
if args is not None and kwargs is not None:
return_value = to_execute(*args, **kwargs)
elif args is not None:
return_value = to_execute(*args)
elif kwargs is not None:
return_value = to_execute(**kwargs)
else:
return_value = to_execute()
returned = True
break
except Exception as error: # pylint: disable=broad-except
if type(error) in catch_exceptions:
cached_error = error
strategy.delay(failover_counter)
failover_counter += 1
continue
else:
_handle_failure(on_failure = on_failure,
error = error)
return
else:
try:
if retry_args is not None and retry_kwargs is not None:
return_value = retry_execute(*retry_args, **retry_kwargs)
elif retry_args is not None:
return_value = retry_execute(*retry_args)
elif retry_kwargs is not None:
return_value = retry_execute(**retry_kwargs)
else:
return_value = retry_execute()
returned = True
break
except Exception as error: # pylint: disable=broad-except
if type(error) in catch_exceptions:
strategy.delay(failover_counter)
cached_error = error
failover_counter += 1
continue
else:
_handle_failure(on_failure = on_failure,
error = error)
return
if not returned:
_handle_failure(on_failure = on_failure,
error = cached_error)
return
elif returned and on_success is not None:
on_success(return_value)
return return_value
