def make_task(func, a, kw):
ft_name = funcname(func) + '-task'
if a:
ft_name = ft_name + ':' + repr(a)
if kw:
ft_name = ft_name + ':' + repr(kw)
return task_class(name=ft_name, func=func, func_args=a, func_kwargs=kw)
def orm_auto_session(method):
''' Decorator to run a method in a session derived from `self.orm`
if a session is not presupplied.
Intended to assist classes with a `.orm` attribute.
See `with_session` for details.
'''
if isgeneratorfunction(method):
def orm_auto_session_wrapper(self, *a, session=None, **kw):
''' Yield from the method with a session.
'''
with using_session(orm=self.orm, session=session) as active_session:
yield from method(self, *a, session=active_session, **kw)
else:
def orm_auto_session_wrapper(self, *a, session=None, **kw):
''' Call the method with a session.
'''
with using_session(orm=self.orm, session=session) as active_session:
return method(self, *a, session=active_session, **kw)
orm_auto_session_wrapper.__name__ = "@orm_auto_session(%s)" % (
funcname(method),
)
orm_auto_session_wrapper.__doc__ = method.__doc__
orm_auto_session_wrapper.__module__ = getattr(method, '__module__', None)
return orm_auto_session_wrapper
def __str__(self):
s = super().__str__() + ":func=%s" % (funcname(self.func), )
if self.fargs:
s += ":fargs=%r" % (self.fargs, )
if self.fkwargs:
s += ":fkwargs=%r" % (self.fkwargs, )
return s
def __init__(self, name, L, actions, outQ):
''' Initialise the Pipeline from `name`, Later instance `L`,
list of filter functions `actions` and output queue `outQ`.
Each action is either a 2-tuple of (sig, functor) or an
object with a .sig attribute and a .functor method returning
a callable.
'''
MultiOpenMixin.__init__(self)
self.name = name
self.later = L
self.queues = [outQ]
# counter tracking items in play
self._busy = TrackingCounter(name="Pipeline<%s>._items" % (name, ))
RHQ = outQ
for index, action in reversed(list(enumerate(actions))):
try:
func_sig, functor = action
except TypeError:
func_sig = action.sig
functor = action.functor(self.later)
pq_name = ":".join((
name,
str(index),
str(func_sig),
funcname(functor),
))
if func_sig == FUNC_ONE_TO_MANY:
PQ = _PipelineStageOneToMany(pq_name, self, functor, RHQ)
elif func_sig == FUNC_ONE_TO_ONE:
PQ = _PipelineStageOneToOne(pq_name, self, functor, RHQ)
elif func_sig == FUNC_SELECTOR:
select_by = functor
def selector(item):
if select_by(item):
yield item
PQ = _PipelineStageOneToMany(pq_name, self, selector, RHQ)
elif func_sig == FUNC_MANY_TO_MANY:
PQ = _PipelineStageManyToMany(pq_name, self, functor, RHQ)
elif func_sig == FUNC_PIPELINE:
PQ = _PipelineStagePipeline(pq_name, self, functor, RHQ)
else:
raise RuntimeError("unimplemented func_sig=%r, functor=%s" %
(func_sig, functor))
PQ.open()
self.queues.insert(0, PQ)
RHQ = PQ
def _defer(self, func, *args, **kwargs):
''' Defer a function via the internal `Later` queue.
Hold opens on `self` to avoid easy shutdown.
'''
self.open()
def with_self():
with self:
return func(*args, **kwargs)
with_self.__name__ = "with_self:" + funcname(func)
LF = self.__funcQ.defer(with_self)
LF.notify(lambda LF: self.close())
return LF
def exc_fold(func, exc_types=None, exc_return=False):
''' Decorator to catch specific exception types and return a defined default value.
'''
def wrapped(*a, **kw):
try:
return func(*a, **kw)
except exc_types as e:
error("%s", e)
return exc_return
wrapped.__name__ = ("@exc_fold[%r=>%r]%s" %
(exc_types, exc_return, funcname(func)))
doc = getattr(func, '__doc__', '')
if doc:
wrapped.__doc__ = wrapped.__name__ + '\n' + doc
return wrapped
def wrapper(*a,
progress=None,
progress_name=None,
progress_total=None,
progress_report_print=None,
**kw):
if progress_name is None:
progress_name = label or funcname(func)
if progress_report_print is None:
progress_report_print = report_print
if progress is None:
upd = Upd()
if not upd.disabled:
progress = Progress(name=progress_name, total=progress_total)
with progress.bar(upd=upd, report_print=progress_report_print):
return func(*a, progress=progress, **kw)
return func(*a, progress=progress, **kw)
def __init__(self, func, name=None, retry_delay=None):
''' Initialise a `LateFunction`.
Parameters:
* `func` is the callable for later execution.
* `name`, if supplied, specifies an identifying name for the `LateFunction`.
* `retry_local`: time delay before retry of this function on RetryError.
Default from `later.retry_delay`.
'''
Result.__init__(self)
self.func = func
if name is None:
name = "LF-%d[%s]" % (seq(), funcname(func))
if retry_delay is None:
retry_delay = DEFAULT_RETRY_DELAY
self.name = name
self.retry_delay = retry_delay
def _after(self, LFs, R, func, *a, **kw):
if not isinstance(LFs, list):
LFs = list(LFs)
if R is None:
R = Result("Later.after(%s)" % (",".join(str(_) for _ in LFs)))
elif not isinstance(R, Result):
raise TypeError(
"Later.after(LFs, R, func, ...): expected Result for R, got %r" %
(R,)
)
def put_func():
''' Function to defer: run `func` and pass its return value to R.put().
'''
R.call(func, *a, **kw)
put_func.__name__ = "%s._after(%r)[func=%s]" % (self, LFs, funcname(func))
return after(LFs, None, lambda: self._defer(put_func))
def bg(
func,
daemon=None,
name=None,
no_start=False,
no_logexc=False,
args=None,
kwargs=None
):
''' Dispatch the callable `func` in its own `Thread`;
return the `Thread`.
Parameters:
* `func`: a callable for the `Thread` target.
* `daemon`: optional argument specifying the `.daemon` attribute.
* `name`: optional argument specifying the `Thread` name,
default: the name of `func`.
* `no_logexc`: if false (default `False`), wrap `func` in `@logexc`.
* `no_start`: optional argument, default `False`.
If true, do not start the `Thread`.
* `args`, `kwargs`: passed to the `Thread` constructor
'''
if name is None:
name = funcname(func)
if args is None:
args = ()
if kwargs is None:
kwargs = {}
##thread_prefix = prefix() + ': ' + name
thread_prefix = name
def thread_body():
with Pfx(thread_prefix):
return func(*args, **kwargs)
T = Thread(name=thread_prefix, target=thread_body)
if not no_logexc:
func = logexc(func)
if daemon is not None:
T.daemon = daemon
if not no_start:
T.start()
return T
def _defer_iterable(self, it, outQ, test_ready=None):
iterate = partial(next, iter(it))
R = Result()
iterationss = [0]
@logexc
def iterate_once():
''' Call `iterate`. Place the result on outQ.
Close the queue at end of iteration or other exception.
Otherwise, requeue ourself to collect the next iteration value.
'''
if test_ready is not None and not test_ready():
raise RetryError("iterate_once: not ready yet")
try:
item = iterate()
except StopIteration:
outQ.close()
R.result = iterationss[0]
except Exception as e: # pylint: disable=broad-except
exception(
"defer_iterable: iterate_once: exception during iteration: %s", e
)
outQ.close()
R.exc_info = sys.exc_info()
else:
iterationss[0] += 1
# put the item onto the output queue
# this may itself defer various tasks (eg in a pipeline)
debug("L.defer_iterable: iterate_once: %s.put(%r)", outQ, item)
outQ.put(item)
# now queue another iteration to run after those defered tasks
self._defer(iterate_once)
iterate_once.__name__ = "%s:next(iter(%s))" % (
funcname(iterate_once), getattr(it, '__name__', repr(it))
)
self._defer(iterate_once)
return R
def locked(func, initial_timeout=10.0, lockattr='_lock'):
''' A decorator for instance methods that must run within a lock.
Decorator keyword arguments:
* `initial_timeout`:
the initial lock attempt timeout;
if this is `>0` and exceeded a warning is issued
and then an indefinite attempt is made.
Default: `2.0`s
* `lockattr`:
the name of the attribute of `self`
which references the lock object.
Default `'_lock'`
'''
citation = "@locked(%s)" % (funcname(func),)
def lockfunc(self, *a, **kw):
''' Obtain the lock and then call `func`.
'''
lock = getattr(self, lockattr)
if initial_timeout > 0 and lock.acquire(timeout=initial_timeout):
try:
return func(self, *a, **kw)
finally:
lock.release()
else:
if initial_timeout > 0:
warning(
"%s: timeout after %gs waiting for %s<%s>.%s, continuing to wait",
citation, initial_timeout,
type(self).__name__, self, lockattr
)
with lock:
return func(self, *a, **kw)
lockfunc.__name__ = citation
lockfunc.__doc__ = getattr(func, '__doc__', '')
return lockfunc
def pfx(func, message=None, message_args=()):
''' General purpose @pfx for generators, methods etc.
Parameters:
* `func`: the function or generator function to decorate
* `message`: optional prefix to use instead of the function name
* `message_args`: optional arguments to embed in the preifx using `%`
Example usage:
@pfx
def f(....):
....
'''
fname = funcname(func)
if message is None:
if message_args:
raise ValueError("no message, but message_args=%r" %
(message_args, ))
if isgeneratorfunction(func):
# persistent in-generator stack to be reused across calls to
# the context manager
saved_stack = []
if message is None:
message = funcname
@contextdecorator
def cmgrdeco(func, a, kw):
''' Context manager to note the entry `Pfx` stack height, append saved
`Pfx` stack from earlier run, then after the iteration step save the
top of the `Pfx` stack for next time.
'''
pfx_stack = Pfx._state.stack
height = len(pfx_stack)
pfx_stack.extend(saved_stack)
with Pfx(message, *message_args):
yield
saved_stack[:] = pfx_stack[height:]
pfx_stack[height:] = []
wrapper = cmgrdeco(func)
else:
if message is None:
def wrapper(*a, **kw):
''' Run function inside `Pfx` context manager.
'''
return pfx_call(func, *a, **kw)
else:
def wrapper(*a, **kw):
''' Run function inside `Pfx` context manager.
'''
with Pfx(message, *message_args):
return func(*a, **kw)
wrapper.__name__ = "@pfx(%s)" % (fname, )
wrapper.__doc__ = func.__doc__
return wrapper