def cursor_contextmanager(processor_id,
injections,
response,
cursor_storage=cursor_storage_context):
cursor_key = key(cursor_name or processor_id,
**apply_context_to_kwargs(context_kwargs, injections))
context_name = '{}_cursor'.format(
cursor_name) if cursor_name else 'cursor'
cursor_value = cursor_storage.get(cursor_key)
if cursor_value is not None:
context = {context_name: cursor_value}
else:
# return empty context if no cursor
context = {}
def cursor_getter():
# get actual cursor value from storage
record = cursor_storage.get(cursor_key)
return record and record.value
def cursor_setter(value):
# update a cursor value in storage
cursor_storage.save(cursor_key, value)
# add cursor property into response
response.set_property(context_name, cursor_getter, cursor_setter)
yield context
def locker_contextmanager(processor_id, injections, lock):
lock_service = lock[lock_category]
lock_key = key(lock_name or processor_id,
**apply_context_to_kwargs(context_kwargs, injections))
locker_obj = Locker(lock_service, lock_key, expire_in)
context = {'locker': locker_obj}
if lock_name:
context['{}_locker'.format(lock_name)] = locker_obj
yield context
def get_message_context(self, program, processor_id, message_dict):
context = self.get_processor_context(program, processor_id)
context['message'] = FrozenMessage(message_dict)
if program.message_mapping:
# Extract mapped message parts from message dictionary
# and update context
context.update(
apply_context_to_kwargs(program.message_mapping,
{'message': message_dict},
ContextPath('message')))
return context
def model_contextmanager(message, response, injections):
model_context = {}
# get model serializer classes from context if any
model_serializers = apply_context_to_kwargs(models, injections)
for model_key, serializer_class in six.iteritems(model_serializers):
serializer = serializer_class(model_key,
injections) # type: IModelSerializer
model_context[model_key] = serializer.deserialize(
message.get(model_key))
response.add_message_filter(
_serialization_filter(model_key, model_context[model_key],
serializer))
yield model_context
def rate_contextmanager(processor_id, rate_counter, injections):
"""
Calculate processor execution rate
:param processor_id: processor id
:type rate_counter: pypipes.context.pool.IContextPool[pypipes.service.rate_counter.IRateCounter]
:type injections: pypipes.context.LazyContextCollection
"""
counter_key = key(
processor_id, **apply_context_to_kwargs(context_kwargs,
injections))
value, expires_in = rate_counter.rate_limit.increment(
counter_key, threshold=rate_threshold)
if value > rate_limit:
raise RateLimitExceededException(retry_in=expires_in)
yield {}
def quota_guard_contextmanager(injections, quota=None):
"""
:type injections: pypipes.context.LazyContextCollection
:type quota: pypipes.context.pool.IContextPool[pypipes.service.quota.IQuota]
:raise: pypipes.exceptions.QuotaExceededException
"""
if quota:
quota_key = (key(
operation_name or '',
**apply_context_to_kwargs(context_kwargs, injections))
if operation_name or context_kwargs else None)
# try to consume from quota
# consume raises a QuotaExceededException when quota is exceeded
quota[quota_name].consume(quota_key)
yield {}
def cache_contextmanager(message, processor_id, response, injections,
cache):
"""
Cache processor response
:type message: pypipes.message.FrozenMessage
:param processor_id: unique processor id
:type response: pypipes.infrastructure.response.IResponseHandler
:type injections: dict
:type cache: pypipes.context.pool.IContextPool[pypipes.service.cache.ICache]
"""
cache_client = cache.processor
key_params = (apply_context_to_kwargs(context_kwargs, injections)
if context_kwargs else message)
cache_key = key(processor_id, **key_params)
messages = cache_client.get(cache_key)
if messages is not None:
# emit cached results
for message in messages:
response.emit_message(message)
# skip message processing
raise DropMessageException('Message hit in cache')
# temp storage for emitted messages
emitted_messages = []
def _filter(msg):
# save emitted message into temp storage
emitted_messages.append(dict(msg))
return msg
response.add_message_filter(_filter)
yield {'cache_key': cache_key}
# save messages into the cache if no exception
def _cache_on_flush(original_flush):
original_flush()
# cache processing result
cache_client.save(cache_key,
emitted_messages,
expires_in=expires_in)
response.extend_flush(_cache_on_flush)
def job_start(message, lock, storage, logger, injections):
job_params = apply_context_to_kwargs(self.kwargs, injections)
job_key = self.create_job_key(self.job_name, job_params)
# ensure job is not started
job_info = storage.job.get_item(job_key)
if job_info and lock.job.get(job_info.value[0]):
logger.warning(
'Job %r %s is still active and cannot be started again.',
self.job_name, job_params)
return
# start new job
job_id = uuid.uuid4().hex
logger.debug('Starting job %s: %s', job_key, job_id)
lock.job.set(job_id, expire_in=self.expires_in)
storage.job.save(job_key, (job_id, job_params),
collections=[self.job_collection])
response = dict(message)
response[self.job_id_key] = job_id
return response
def create_logger_adapter(injections):
return logging.LoggerAdapter(logger,
extra=apply_context_to_kwargs(
extras, injections))
lookup_path = context.key1.unknown | context.key2.key2_3.unknown | context.key2.key2_3.key2_3_2
print(lookup_path, '=', lookup_path(context_dict))
lookup_path = context.key2 & context.key2.key2_3 & context.key2.key2_3.key2_3_2
print(lookup_path, '=', lookup_path(context_dict))
# bitwise operations between ContextPath and value of other types are also supported
lookup_path = context.key1.unknown | 0
print(lookup_path, '=', lookup_path(context_dict))
lookup_path = context.key2 & True
print(lookup_path, '=', lookup_path(context_dict))
# apply_context_to_kwargs lookups for key's value in context if value is IContextLookup
kwargs = dict(a=1, b=2, c=context.key1)
print(kwargs, '=', apply_context_to_kwargs(kwargs, context_dict))
# if context path is empty, apply_context_to_kwargs uses key name as default context path
kwargs = dict(key1=context, key2=context)
print(kwargs, '=', apply_context_to_kwargs(kwargs, context_dict))
# you can specify which context is default
kwargs = dict(key1=context.key1, key2_1=context, key2_2=context)
print(
kwargs, '=',
apply_context_to_kwargs(kwargs,
context_dict,
default_context_path=context.key2))
# if context dict is a configuration you may use special config path adapter
# that automatically converts result into Config object
def test_apply_context_to_kwargs(context_dict, input, default, expected):
assert apply_context_to_kwargs(input, context_dict, default_context_path=default) == expected
def extend_message(message, injections):
update_values = apply_context_to_kwargs(
dict(extension or {}, **kwargs), injections)
return dict(message, **update_values)
def send_message(injections):
return apply_context_to_kwargs(dict(message or {}, **kwargs),
injections)
def __call__(self, context_dict):
return 'MyCustomContext: {}'.format(context_dict['key2'] +
context_dict['key3'])
lazy_context = LazyContextCollection(key1=MyContextFactory(), key2=2, key3=3)
print('lazy_context = ', lazy_context)
print('lazy_context.get("key1") = ',
LazyContextCollection(lazy_context).get('key1'))
print('lazy_context["key1"] = ', LazyContextCollection(lazy_context)['key1'])
# this works in apply_context_to_kwargs as well
kwargs = dict(key1=context.key1)
print(kwargs, '=',
apply_context_to_kwargs(kwargs, LazyContextCollection(lazy_context)))
# if kay not found in a LazyContextCollection, the KeyError message will contain a list
# of available context for better error tracing.
try:
LazyContextCollection(lazy_context)['unknown']
except KeyError as e:
print('KeyError: ', e)
# Also LazyContextCollection ensures that context initialization has no initialization loops
# For example, if 'key2' is also a lazy context with same factory,
# key1 context initialization will have a loop
lazy_context['key2'] = MyContextFactory()
try:
LazyContextCollection(lazy_context)['key1']
except KeyError as e:
def _define(injections):
yield apply_context_to_kwargs(kwargs, injections)
def _cached_lazy_context(cache, lock, injections):
"""
Get context from cache or create a new one
:type cache: IContextPool[ICache]
:type lock: IContextPool[ILock]
:param injections: context collection
:return: context object
"""
context_cache = cache.context
context_lock = lock.context
context_key = key(context_name,
apply_context_to_kwargs(key_params, injections))
def _create_and_save_context():
new_context = context_factory(injections)
if isinstance(new_context, MetaValue):
# factory may override default expiration and other predefined params
_expires_in = int(
new_context.metadata.get('expires_in', expires_in))
_lock_on = int(new_context.metadata.get('lock_on', 0))
new_context = new_context.value
else:
_expires_in = expires_in
_lock_on = False
if _lock_on:
# context factory demands to lock it for some time
# that may be caused by rate limit or other resource limitations
logger.warning(
'Context factory for %s is locked for %s seconds',
context_key, _lock_on)
context_lock.set(context_key, expire_in=_lock_on)
if new_context is None:
raise AssertionError('Context value for %r is not available',
context_key)
# save new value into the cache
context_cache.save(context_key,
(new_context, get_refresh_at(_expires_in)),
expires_in=_expires_in)
return new_context
tries = 10 # max count of tries to get the context
while tries > 0:
tries -= 1
value = context_cache.get(context_key)
if value:
context, refresh_at = value
if (refresh_at and datetime.utcnow() > refresh_at
and context_lock.acquire(context_key,
expire_in=generation_time)):
# current context value is still valid
# but it's a good time to prepare a new one in advance
# that will prevents blocking of other processors
context = _create_and_save_context()
return context
else:
if context_lock.acquire(context_key,
expire_in=generation_time):
return _create_and_save_context()
else:
# some other processor is generating the context right now
# check when the new context will be ready
lock_expires_in = context_lock.get(context_key)
if lock_expires_in:
if lock_expires_in > generation_time:
# the context factory locked itself for a long time
raise RetryMessageException(
'Context factory for {!r} is locked'.format(
context_name),
retry_in=lock_expires_in)
# sleep a little and check again
sleep(1)
# context is still not ready after all tries
# retry the message processing some later
logger.error('Failed to create context: %s', context_name)
raise RetryMessageException(
'Context {!r} creation took too much time'.format(context_name),
retry_in=60)
