def cursor_contextmanager(processor_id, injections, response, cursor_storage=cursor_storage_context): cursor_key = key(cursor_name or processor_id, **apply_context_to_kwargs(context_kwargs, injections)) context_name = '{}_cursor'.format( cursor_name) if cursor_name else 'cursor' cursor_value = cursor_storage.get(cursor_key) if cursor_value is not None: context = {context_name: cursor_value} else: # return empty context if no cursor context = {} def cursor_getter(): # get actual cursor value from storage record = cursor_storage.get(cursor_key) return record and record.value def cursor_setter(value): # update a cursor value in storage cursor_storage.save(cursor_key, value) # add cursor property into response response.set_property(context_name, cursor_getter, cursor_setter) yield context
def locker_contextmanager(processor_id, injections, lock): lock_service = lock[lock_category] lock_key = key(lock_name or processor_id, **apply_context_to_kwargs(context_kwargs, injections)) locker_obj = Locker(lock_service, lock_key, expire_in) context = {'locker': locker_obj} if lock_name: context['{}_locker'.format(lock_name)] = locker_obj yield context
def get_message_context(self, program, processor_id, message_dict): context = self.get_processor_context(program, processor_id) context['message'] = FrozenMessage(message_dict) if program.message_mapping: # Extract mapped message parts from message dictionary # and update context context.update( apply_context_to_kwargs(program.message_mapping, {'message': message_dict}, ContextPath('message'))) return context
def model_contextmanager(message, response, injections): model_context = {} # get model serializer classes from context if any model_serializers = apply_context_to_kwargs(models, injections) for model_key, serializer_class in six.iteritems(model_serializers): serializer = serializer_class(model_key, injections) # type: IModelSerializer model_context[model_key] = serializer.deserialize( message.get(model_key)) response.add_message_filter( _serialization_filter(model_key, model_context[model_key], serializer)) yield model_context
def rate_contextmanager(processor_id, rate_counter, injections): """ Calculate processor execution rate :param processor_id: processor id :type rate_counter: pypipes.context.pool.IContextPool[pypipes.service.rate_counter.IRateCounter] :type injections: pypipes.context.LazyContextCollection """ counter_key = key( processor_id, **apply_context_to_kwargs(context_kwargs, injections)) value, expires_in = rate_counter.rate_limit.increment( counter_key, threshold=rate_threshold) if value > rate_limit: raise RateLimitExceededException(retry_in=expires_in) yield {}
def quota_guard_contextmanager(injections, quota=None): """ :type injections: pypipes.context.LazyContextCollection :type quota: pypipes.context.pool.IContextPool[pypipes.service.quota.IQuota] :raise: pypipes.exceptions.QuotaExceededException """ if quota: quota_key = (key( operation_name or '', **apply_context_to_kwargs(context_kwargs, injections)) if operation_name or context_kwargs else None) # try to consume from quota # consume raises a QuotaExceededException when quota is exceeded quota[quota_name].consume(quota_key) yield {}
def cache_contextmanager(message, processor_id, response, injections, cache): """ Cache processor response :type message: pypipes.message.FrozenMessage :param processor_id: unique processor id :type response: pypipes.infrastructure.response.IResponseHandler :type injections: dict :type cache: pypipes.context.pool.IContextPool[pypipes.service.cache.ICache] """ cache_client = cache.processor key_params = (apply_context_to_kwargs(context_kwargs, injections) if context_kwargs else message) cache_key = key(processor_id, **key_params) messages = cache_client.get(cache_key) if messages is not None: # emit cached results for message in messages: response.emit_message(message) # skip message processing raise DropMessageException('Message hit in cache') # temp storage for emitted messages emitted_messages = [] def _filter(msg): # save emitted message into temp storage emitted_messages.append(dict(msg)) return msg response.add_message_filter(_filter) yield {'cache_key': cache_key} # save messages into the cache if no exception def _cache_on_flush(original_flush): original_flush() # cache processing result cache_client.save(cache_key, emitted_messages, expires_in=expires_in) response.extend_flush(_cache_on_flush)
def job_start(message, lock, storage, logger, injections): job_params = apply_context_to_kwargs(self.kwargs, injections) job_key = self.create_job_key(self.job_name, job_params) # ensure job is not started job_info = storage.job.get_item(job_key) if job_info and lock.job.get(job_info.value[0]): logger.warning( 'Job %r %s is still active and cannot be started again.', self.job_name, job_params) return # start new job job_id = uuid.uuid4().hex logger.debug('Starting job %s: %s', job_key, job_id) lock.job.set(job_id, expire_in=self.expires_in) storage.job.save(job_key, (job_id, job_params), collections=[self.job_collection]) response = dict(message) response[self.job_id_key] = job_id return response
def create_logger_adapter(injections): return logging.LoggerAdapter(logger, extra=apply_context_to_kwargs( extras, injections))
lookup_path = context.key1.unknown | context.key2.key2_3.unknown | context.key2.key2_3.key2_3_2 print(lookup_path, '=', lookup_path(context_dict)) lookup_path = context.key2 & context.key2.key2_3 & context.key2.key2_3.key2_3_2 print(lookup_path, '=', lookup_path(context_dict)) # bitwise operations between ContextPath and value of other types are also supported lookup_path = context.key1.unknown | 0 print(lookup_path, '=', lookup_path(context_dict)) lookup_path = context.key2 & True print(lookup_path, '=', lookup_path(context_dict)) # apply_context_to_kwargs lookups for key's value in context if value is IContextLookup kwargs = dict(a=1, b=2, c=context.key1) print(kwargs, '=', apply_context_to_kwargs(kwargs, context_dict)) # if context path is empty, apply_context_to_kwargs uses key name as default context path kwargs = dict(key1=context, key2=context) print(kwargs, '=', apply_context_to_kwargs(kwargs, context_dict)) # you can specify which context is default kwargs = dict(key1=context.key1, key2_1=context, key2_2=context) print( kwargs, '=', apply_context_to_kwargs(kwargs, context_dict, default_context_path=context.key2)) # if context dict is a configuration you may use special config path adapter # that automatically converts result into Config object
def test_apply_context_to_kwargs(context_dict, input, default, expected): assert apply_context_to_kwargs(input, context_dict, default_context_path=default) == expected
def extend_message(message, injections): update_values = apply_context_to_kwargs( dict(extension or {}, **kwargs), injections) return dict(message, **update_values)
def send_message(injections): return apply_context_to_kwargs(dict(message or {}, **kwargs), injections)
def __call__(self, context_dict): return 'MyCustomContext: {}'.format(context_dict['key2'] + context_dict['key3']) lazy_context = LazyContextCollection(key1=MyContextFactory(), key2=2, key3=3) print('lazy_context = ', lazy_context) print('lazy_context.get("key1") = ', LazyContextCollection(lazy_context).get('key1')) print('lazy_context["key1"] = ', LazyContextCollection(lazy_context)['key1']) # this works in apply_context_to_kwargs as well kwargs = dict(key1=context.key1) print(kwargs, '=', apply_context_to_kwargs(kwargs, LazyContextCollection(lazy_context))) # if kay not found in a LazyContextCollection, the KeyError message will contain a list # of available context for better error tracing. try: LazyContextCollection(lazy_context)['unknown'] except KeyError as e: print('KeyError: ', e) # Also LazyContextCollection ensures that context initialization has no initialization loops # For example, if 'key2' is also a lazy context with same factory, # key1 context initialization will have a loop lazy_context['key2'] = MyContextFactory() try: LazyContextCollection(lazy_context)['key1'] except KeyError as e:
def _define(injections): yield apply_context_to_kwargs(kwargs, injections)
def _cached_lazy_context(cache, lock, injections): """ Get context from cache or create a new one :type cache: IContextPool[ICache] :type lock: IContextPool[ILock] :param injections: context collection :return: context object """ context_cache = cache.context context_lock = lock.context context_key = key(context_name, apply_context_to_kwargs(key_params, injections)) def _create_and_save_context(): new_context = context_factory(injections) if isinstance(new_context, MetaValue): # factory may override default expiration and other predefined params _expires_in = int( new_context.metadata.get('expires_in', expires_in)) _lock_on = int(new_context.metadata.get('lock_on', 0)) new_context = new_context.value else: _expires_in = expires_in _lock_on = False if _lock_on: # context factory demands to lock it for some time # that may be caused by rate limit or other resource limitations logger.warning( 'Context factory for %s is locked for %s seconds', context_key, _lock_on) context_lock.set(context_key, expire_in=_lock_on) if new_context is None: raise AssertionError('Context value for %r is not available', context_key) # save new value into the cache context_cache.save(context_key, (new_context, get_refresh_at(_expires_in)), expires_in=_expires_in) return new_context tries = 10 # max count of tries to get the context while tries > 0: tries -= 1 value = context_cache.get(context_key) if value: context, refresh_at = value if (refresh_at and datetime.utcnow() > refresh_at and context_lock.acquire(context_key, expire_in=generation_time)): # current context value is still valid # but it's a good time to prepare a new one in advance # that will prevents blocking of other processors context = _create_and_save_context() return context else: if context_lock.acquire(context_key, expire_in=generation_time): return _create_and_save_context() else: # some other processor is generating the context right now # check when the new context will be ready lock_expires_in = context_lock.get(context_key) if lock_expires_in: if lock_expires_in > generation_time: # the context factory locked itself for a long time raise RetryMessageException( 'Context factory for {!r} is locked'.format( context_name), retry_in=lock_expires_in) # sleep a little and check again sleep(1) # context is still not ready after all tries # retry the message processing some later logger.error('Failed to create context: %s', context_name) raise RetryMessageException( 'Context {!r} creation took too much time'.format(context_name), retry_in=60)