def __init__(self, factory_func):
self._factory_func = injections_handler(factory_func)
# call function with positional args. Arguments have priority over value taken from context
func_with_injections('arg1', 'arg2')
# call function with keywords. They also have priority over default values.
func_with_injections(context1='keyword1', context3='keyword3')
# function raises TypeError if some context is not provided.
try:
func_with_injections()
except TypeError as e:
print('TypeError:', e)
# another way to use a parameter injection is creating of an injection handler that will
# translate input context into function parameters
func_with_injections = injections_handler(target_function)
# call injection handler with context dictionary.
# Context should have some values for each func parameter otherwise TypeError is raised
try:
func_with_injections(context)
except TypeError as e:
print('TypeError:', e)
# append missing context into collection and try again
context['context1'] = 'context_value1'
func_with_injections(context)
# target function receives an entire context if some parameter name is 'injections'
@injections_handler
def cached_lazy_context(_gen_func=None,
expires_in=None,
generation_time=5,
**key_params):
"""
Create a lazy context that handles context generation, caching and refreshing
:param _gen_func: Context factory that may use context injections
:type _gen_func: (ANY) -> object
:param expires_in: context cache expiration time in seconds
:param generation_time: time needed to generate a new context
:param key_params: context key. The factory will create a separate context per context key
:rtype: pypipes.context.factory.LazyContext
"""
if _gen_func is None:
return partial(cached_lazy_context,
expires_in=expires_in,
generate_in=generation_time,
**key_params)
assert generation_time > 0
context_name = _gen_func.__name__
context_factory = injections_handler(_gen_func)
def get_refresh_at(_expires_in):
# returns time when the context may be refreshed
if _expires_in:
return datetime.utcnow() + timedelta(
seconds=(_expires_in - min(_expires_in / 2, generation_time)))
@lazy_context
def _cached_lazy_context(cache, lock, injections):
"""
Get context from cache or create a new one
:type cache: IContextPool[ICache]
:type lock: IContextPool[ILock]
:param injections: context collection
:return: context object
"""
context_cache = cache.context
context_lock = lock.context
context_key = key(context_name,
apply_context_to_kwargs(key_params, injections))
def _create_and_save_context():
new_context = context_factory(injections)
if isinstance(new_context, MetaValue):
# factory may override default expiration and other predefined params
_expires_in = int(
new_context.metadata.get('expires_in', expires_in))
_lock_on = int(new_context.metadata.get('lock_on', 0))
new_context = new_context.value
else:
_expires_in = expires_in
_lock_on = False
if _lock_on:
# context factory demands to lock it for some time
# that may be caused by rate limit or other resource limitations
logger.warning(
'Context factory for %s is locked for %s seconds',
context_key, _lock_on)
context_lock.set(context_key, expire_in=_lock_on)
if new_context is None:
raise AssertionError('Context value for %r is not available',
context_key)
# save new value into the cache
context_cache.save(context_key,
(new_context, get_refresh_at(_expires_in)),
expires_in=_expires_in)
return new_context
tries = 10 # max count of tries to get the context
while tries > 0:
tries -= 1
value = context_cache.get(context_key)
if value:
context, refresh_at = value
if (refresh_at and datetime.utcnow() > refresh_at
and context_lock.acquire(context_key,
expire_in=generation_time)):
# current context value is still valid
# but it's a good time to prepare a new one in advance
# that will prevents blocking of other processors
context = _create_and_save_context()
return context
else:
if context_lock.acquire(context_key,
expire_in=generation_time):
return _create_and_save_context()
else:
# some other processor is generating the context right now
# check when the new context will be ready
lock_expires_in = context_lock.get(context_key)
if lock_expires_in:
if lock_expires_in > generation_time:
# the context factory locked itself for a long time
raise RetryMessageException(
'Context factory for {!r} is locked'.format(
context_name),
retry_in=lock_expires_in)
# sleep a little and check again
sleep(1)
# context is still not ready after all tries
# retry the message processing some later
logger.error('Failed to create context: %s', context_name)
raise RetryMessageException(
'Context {!r} creation took too much time'.format(context_name),
retry_in=60)
return _cached_lazy_context
def __init__(self, processor_func, context_managers=None, events=None):
MultiContextManager.__init__(self, context_managers)
Processor.__init__(self, events)
self.processor_func = processor_func
self.injections_handler = injections_handler(processor_func)
def __init__(self, gen_func):
super(PipeContextManager, self).__init__()
self._gen_func = injections_handler(gen_func)