def scheduler_stop(processor_id, message, response, injections):
split_by_value = split_by and split_by(
injections) # initialize lazy factory
complex_scheduler_id = scheduler_id
if split_by_value is not None:
complex_scheduler_id = (key(scheduler_id, split_by_value)
if scheduler_id else key(
processor_id, split_by_value))
response.stop_scheduler(complex_scheduler_id)
return message
def scheduler(processor_id, message, response, injections):
# scheduler id might be formatted with data from message or other context
# that allows creating a separate scheduler per specific context value
split_by_value = split_by and split_by(
injections) # initialize lazy factory
complex_scheduler_id = scheduler_id
if split_by_value is not None:
complex_scheduler_id = (key(scheduler_id, split_by_value)
if scheduler_id else key(
processor_id, split_by_value))
scheduler_period = message.pop('_scheduler_period', period)
if scheduler_period:
response.schedule_message(message,
scheduler_id=complex_scheduler_id,
period=scheduler_period)
def __init__(self, job_name, expires_in=None, **kwargs):
self.job_name = job_name
self.kwargs = kwargs
self.expires_in = expires_in or self.MAX_JOB_INACTIVE_PERIOD
self.event_name = self.JOB_EVENT_TEMPLATE.format(job_name)
self.job_collection = key('started_jobs', job_name)
self.job_id_key = '_job_{}'.format(job_name)
def cursor_contextmanager(processor_id,
injections,
response,
cursor_storage=cursor_storage_context):
cursor_key = key(cursor_name or processor_id,
**apply_context_to_kwargs(context_kwargs, injections))
context_name = '{}_cursor'.format(
cursor_name) if cursor_name else 'cursor'
cursor_value = cursor_storage.get(cursor_key)
if cursor_value is not None:
context = {context_name: cursor_value}
else:
# return empty context if no cursor
context = {}
def cursor_getter():
# get actual cursor value from storage
record = cursor_storage.get(cursor_key)
return record and record.value
def cursor_setter(value):
# update a cursor value in storage
cursor_storage.save(cursor_key, value)
# add cursor property into response
response.set_property(context_name, cursor_getter, cursor_setter)
yield context
def save(self, cursor_name, value):
versioned_name = key(cursor_name, self._version)
self._storage.save(versioned_name,
value,
collections=[
cursor_name, self._all_collection,
self._version_collection
])
def locker_contextmanager(processor_id, injections, lock):
lock_service = lock[lock_category]
lock_key = key(lock_name or processor_id,
**apply_context_to_kwargs(context_kwargs, injections))
locker_obj = Locker(lock_service, lock_key, expire_in)
context = {'locker': locker_obj}
if lock_name:
context['{}_locker'.format(lock_name)] = locker_obj
yield context
def trigger_scheduler(self, program, scheduler_id):
"""
:type self: CelerySchedulerMixIn, BaseCeleryInf
"""
full_scheduler_id = self._scheduler_storage_id(program, scheduler_id)
if self.scheduler_storage.get_item(full_scheduler_id):
# this lock is a trigger for scheduler task
self.program_lock.set(key('scheduler', full_scheduler_id),
expire_in=self.MAX_SCHEDULER_COUNTDOWN * 2)
return True
def wrapper(self, *args, **kwargs):
global func_key_builder
_key_builder = key_builder or func_key_builder
func_key = _key_builder.format_key(self.__class__.__name__, func.__name__,
key(*args, **kwargs))
if self.cache:
result = self.cache.get(func_key)
if result:
return result
result = func(self, *args, **kwargs)
if self.cache:
self.cache.save(func_key, result, expires_in=expires_in)
return result
def rate_contextmanager(processor_id, rate_counter, injections):
"""
Calculate processor execution rate
:param processor_id: processor id
:type rate_counter: pypipes.context.pool.IContextPool[pypipes.service.rate_counter.IRateCounter]
:type injections: pypipes.context.LazyContextCollection
"""
counter_key = key(
processor_id, **apply_context_to_kwargs(context_kwargs,
injections))
value, expires_in = rate_counter.rate_limit.increment(
counter_key, threshold=rate_threshold)
if value > rate_limit:
raise RateLimitExceededException(retry_in=expires_in)
yield {}
def quota_guard_contextmanager(injections, quota=None):
"""
:type injections: pypipes.context.LazyContextCollection
:type quota: pypipes.context.pool.IContextPool[pypipes.service.quota.IQuota]
:raise: pypipes.exceptions.QuotaExceededException
"""
if quota:
quota_key = (key(
operation_name or '',
**apply_context_to_kwargs(context_kwargs, injections))
if operation_name or context_kwargs else None)
# try to consume from quota
# consume raises a QuotaExceededException when quota is exceeded
quota[quota_name].consume(quota_key)
yield {}
def cache_contextmanager(message, processor_id, response, injections,
cache):
"""
Cache processor response
:type message: pypipes.message.FrozenMessage
:param processor_id: unique processor id
:type response: pypipes.infrastructure.response.IResponseHandler
:type injections: dict
:type cache: pypipes.context.pool.IContextPool[pypipes.service.cache.ICache]
"""
cache_client = cache.processor
key_params = (apply_context_to_kwargs(context_kwargs, injections)
if context_kwargs else message)
cache_key = key(processor_id, **key_params)
messages = cache_client.get(cache_key)
if messages is not None:
# emit cached results
for message in messages:
response.emit_message(message)
# skip message processing
raise DropMessageException('Message hit in cache')
# temp storage for emitted messages
emitted_messages = []
def _filter(msg):
# save emitted message into temp storage
emitted_messages.append(dict(msg))
return msg
response.add_message_filter(_filter)
yield {'cache_key': cache_key}
# save messages into the cache if no exception
def _cache_on_flush(original_flush):
original_flush()
# cache processing result
cache_client.save(cache_key,
emitted_messages,
expires_in=expires_in)
response.extend_flush(_cache_on_flush)
def quota_guard(quota_name=None,
suspend=True,
operation_name=None,
**context_kwargs):
"""
Consume from quota on each message processing begin
:param quota_name: name of quota
:param operation_name: name of operation if you have a separate quota per operation
:param context_kwargs: addition quota sub-key parameters
:param suspend: suspend processing if quota is exceeded, otherwise drop messages
:return: pipe_contextmanager
:rtype: pipe_contextmanager
"""
lock_name = key(quota_name,
operation_name) if operation_name else quota_name
@suspended_guard(lock_name,
lock_category='quota_guard',
retry_if_locked=suspend,
**context_kwargs)
@pipe_contextmanager
def quota_guard_contextmanager(injections, quota=None):
"""
:type injections: pypipes.context.LazyContextCollection
:type quota: pypipes.context.pool.IContextPool[pypipes.service.quota.IQuota]
:raise: pypipes.exceptions.QuotaExceededException
"""
if quota:
quota_key = (key(
operation_name or '',
**apply_context_to_kwargs(context_kwargs, injections))
if operation_name or context_kwargs else None)
# try to consume from quota
# consume raises a QuotaExceededException when quota is exceeded
quota[quota_name].consume(quota_key)
yield {}
return quota_guard_contextmanager
def _cached_lazy_context(cache, lock, injections):
"""
Get context from cache or create a new one
:type cache: IContextPool[ICache]
:type lock: IContextPool[ILock]
:param injections: context collection
:return: context object
"""
context_cache = cache.context
context_lock = lock.context
context_key = key(context_name,
apply_context_to_kwargs(key_params, injections))
def _create_and_save_context():
new_context = context_factory(injections)
if isinstance(new_context, MetaValue):
# factory may override default expiration and other predefined params
_expires_in = int(
new_context.metadata.get('expires_in', expires_in))
_lock_on = int(new_context.metadata.get('lock_on', 0))
new_context = new_context.value
else:
_expires_in = expires_in
_lock_on = False
if _lock_on:
# context factory demands to lock it for some time
# that may be caused by rate limit or other resource limitations
logger.warning(
'Context factory for %s is locked for %s seconds',
context_key, _lock_on)
context_lock.set(context_key, expire_in=_lock_on)
if new_context is None:
raise AssertionError('Context value for %r is not available',
context_key)
# save new value into the cache
context_cache.save(context_key,
(new_context, get_refresh_at(_expires_in)),
expires_in=_expires_in)
return new_context
tries = 10 # max count of tries to get the context
while tries > 0:
tries -= 1
value = context_cache.get(context_key)
if value:
context, refresh_at = value
if (refresh_at and datetime.utcnow() > refresh_at
and context_lock.acquire(context_key,
expire_in=generation_time)):
# current context value is still valid
# but it's a good time to prepare a new one in advance
# that will prevents blocking of other processors
context = _create_and_save_context()
return context
else:
if context_lock.acquire(context_key,
expire_in=generation_time):
return _create_and_save_context()
else:
# some other processor is generating the context right now
# check when the new context will be ready
lock_expires_in = context_lock.get(context_key)
if lock_expires_in:
if lock_expires_in > generation_time:
# the context factory locked itself for a long time
raise RetryMessageException(
'Context factory for {!r} is locked'.format(
context_name),
retry_in=lock_expires_in)
# sleep a little and check again
sleep(1)
# context is still not ready after all tries
# retry the message processing some later
logger.error('Failed to create context: %s', context_name)
raise RetryMessageException(
'Context {!r} creation took too much time'.format(context_name),
retry_in=60)
def create_job_key(name, params):
return key(name, **params)
def scheduler_task(self, task, program_id, scheduler_id, token,
processor_id, message, repeat_period, start_time):
"""
This task repeats itself periodically to implement a scheduler
:type self: CelerySchedulerMixIn, BaseCeleryInf
:param task: celery task reference
:type task: celery.task.Task
:param program_id: program id
:param scheduler_id: scheduler id
:param token: unique scheduler token
:param processor_id: processor id
:param message: scheduled message
:param repeat_period: repeat period of the scheduler
:param start_time: time when the task have to be started
"""
countdown = 0
try:
logger.debug('Start scheduler_task for: %s',
(program_id, token, processor_id, message,
repeat_period, start_time))
program = self.get_program(program_id)
if not program:
logger.warning(
'received a scheduled message for unknown program: %s',
program_id)
return
if not self.program_lock.get(key('started', program_id)):
# the program is already stopped
return
full_scheduler_id = self._scheduler_storage_id(
program, scheduler_id)
saved_token = self.scheduler_storage.get_item(full_scheduler_id)
if not saved_token or saved_token.value != token:
# scheduler was updated. Current scheduler task is not actual anymore
return
if self.program_lock.release(key('scheduler', full_scheduler_id)):
logger.info('Scheduler %s activated on demand', scheduler_id)
elif start_time:
countdown = self._get_countdown(start_time)
if countdown and countdown > self.MIN_SCHEDULER_PRECISION:
logger.debug(
'Start time is not reached yet, wait next %s seconds',
countdown)
raise task.retry(countdown=countdown)
self.send_message(program, processor_id, message)
if repeat_period:
# restart this task in repeat_periods
start_time = datetime.now() + timedelta(seconds=repeat_period)
countdown = self._get_countdown(start_time)
logger.debug(
'Schedule next task execution at %s, next tick in %s seconds',
start_time, countdown)
task.request.retries = 0 # drop retries counter
raise task.retry(kwargs=dict(task.request.kwargs,
start_time=start_time),
countdown=countdown)
except MaxRetriesExceededError:
# just in case, to be sure, that max retry error is not possible here
logger.error('MaxRetriesExceededError happened in scheduler: %s',
scheduler_id)
task.request.retries = 0
raise task.retry(countdown=countdown)
except TaskPredicate:
# celery service exceptions like Retry
raise
except Exception:
# in case of any other exception, retry the task immediately
logger.exception(
'Scheduler task raised an exception for '
'args:%r, kwargs:%r', task.request.args, task.request.kwargs)
raise task.retry(countdown=5)
def _scheduler_collection_id(program):
return key('schedulers', program.id)
def _scheduler_storage_id(program, scheduler_id):
return key(program.id, scheduler_id)
def try_stop_program(self, program):
if self.program_lock.release(key('started', program.id)):
return super(BaseCeleryInf, self).try_stop_program(program)
return False