def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
"""Like pydispatcher.robust.sendRobust but it also logs errors and returns
Failures instead of exceptions.
"""
dont_log = named.pop("dont_log", _IgnoredException)
spider = named.get("spider", None)
responses = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
try:
response = robustApply(receiver, signal=signal, sender=sender, *arguments, **named)
if isinstance(response, Deferred):
logger.error(
"Cannot return deferreds from signal handler: %(receiver)s",
{"receiver": receiver},
extra={"spider": spider},
)
except dont_log:
result = Failure()
except Exception:
result = Failure()
logger.error(
"Error caught on signal handler: %(receiver)s",
{"receiver": receiver},
exc_info=True,
extra={"spider": spider},
)
else:
result = response
responses.append((receiver, result))
return responses
def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
"""Like pydispatcher.robust.sendRobust but it also logs errors and returns
Failures instead of exceptions.
"""
dont_log = (named.pop('dont_log', _IgnoredException), StopDownload)
spider = named.get('spider', None)
responses = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
try:
response = robustApply(receiver,
signal=signal,
sender=sender,
*arguments,
**named)
if isinstance(response, Deferred):
logger.error(
"Cannot return deferreds from signal handler: %(receiver)s",
{'receiver': receiver},
extra={'spider': spider})
except dont_log:
result = Failure()
except Exception:
result = Failure()
logger.error("Error caught on signal handler: %(receiver)s",
{'receiver': receiver},
exc_info=True,
extra={'spider': spider})
else:
result = response
responses.append((receiver, result))
return responses
def show_receivers(self, sender=None, signal=None):
sender = sender if sender is not None else dispatcher.Any
signal = signal if signal is not None else dispatcher.Any
print("*** live receivers ***")
for rec in dispatcher.liveReceivers(dispatcher.getReceivers(sender, signal)):
print("receiver -->", rec)
print("*** end live receivers ***")
def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
"""Like send_catch_log but supports returning deferreds on signal handlers.
Returns a deferred that gets fired once all signal handlers deferreds were
fired.
"""
def logerror(failure, recv):
if dont_log is None or not isinstance(failure.value, dont_log):
logger.error(
"Error caught on signal handler: %(receiver)s",
{"receiver": recv},
exc_info=failure_to_exc_info(failure),
extra={"spider": spider},
)
return failure
dont_log = named.pop("dont_log", None)
spider = named.get("spider", None)
dfds = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
d = maybeDeferred(robustApply, receiver, signal=signal, sender=sender, *arguments, **named)
d.addErrback(logerror, receiver)
d.addBoth(lambda result: (receiver, result))
dfds.append(d)
d = DeferredList(dfds)
d.addCallback(lambda out: [x[1] for x in out])
return d
def _removeCurrentCallbacks(
cls,
key,
node = None,
capture = 0,
):
"""De-register current callbacks, return previous callback
Note:
There should only ever be one receiver registered
this way, so should always be either None or the
previous callback.
"""
metaKey = (cls.type,capture,key)
# remove all current watchers...
receivers = dispatcher.liveReceivers(
dispatcher.getReceivers(
sender=node,
signal=metaKey,
)
)
receiver = None
for receiver in receivers:
if __debug__:
log.info( """Disconnecting receiver for key %(metaKey)r for node %(node)r -> %(receiver)s"""%locals())
dispatcher.disconnect(
receiver,
signal = metaKey,
sender=node,
)
assert len(dispatcher.getReceivers(
sender=node,
signal=metaKey,
)) == 0, """Event callback de-registration failed: %(cls)s %(key)s %(node)r"""%locals()
return receiver
def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
"""Like send_catch_log but supports returning deferreds on signal handlers.
Returns a deferred that gets fired once all signal handlers deferreds were
fired.
"""
def logerror(failure, recv):
if dont_log is None or not isinstance(failure.value, dont_log):
logger.error("Error caught on signal handler: %(receiver)s",
{'receiver': recv},
exc_info=failure_to_exc_info(failure),
extra={'spider': spider})
return failure
dont_log = named.pop('dont_log', None)
spider = named.get('spider', None)
dfds = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
d = maybeDeferred_coro(robustApply,
receiver,
signal=signal,
sender=sender,
*arguments,
**named)
d.addErrback(logerror, receiver)
d.addBoth(lambda result: (receiver, result))
dfds.append(d)
d = DeferredList(dfds)
d.addCallback(lambda out: [x[1] for x in out])
return d
def send_deferred(signal=Any, sender=Anonymous, *arguments, **named):
"""Like pydispatcher.send but supports returning deferreds on signal handlers.
Returns a deferred that gets fired once all signal handlers deferreds were
fired.
If one signal handler fails, the deffered errback is called.
"""
spider = named.get('spider', None)
dfds = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
d = maybeDeferred(robustApply,
receiver,
signal=signal,
sender=sender,
*arguments,
**named)
d.addCallback(lambda result: (receiver, result))
dfds.append(d)
d = DeferredList(dfds, consumeErrors=True, fireOnOneErrback=True)
def unwrap_firsterror(err):
failure = err.trap(FirstError)
return err.value.subFailure
d.addErrback(unwrap_firsterror)
d.addCallback(lambda out: [x[1] for x in out])
return d
def _get_receivers(self, sender, signal):
"""
Return receivers for specified sender and signal
:return: generator over receivers
"""
return dispatcher.liveReceivers(
dispatcher.getAllReceivers(sender, signal))
def disconnect_all(self, signal, **kwargs) -> None:
"""
将所有接收器断开给定信号的连接
:param signal:
:param kwargs:
:return:
"""
for receiver in dispatcher.liveReceivers(
dispatcher.getAllReceivers(self.send, signal)):
dispatcher.disconnect(receiver, signal, self.sender, **kwargs)
async def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
"""Like send_catch_log but supports returning deferreds on signal handlers.
Returns a deferred that gets fired once all signal handlers deferreds were
fired.
"""
dfds = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
dfds.append(asyncio.create_task(
robustApplyWrap(robustApply, receiver, signal=signal, sender=sender, *arguments, **named)))
res = await asyncio.gather(*dfds)
return res
def sendRobust(
signal=Any,
sender=Anonymous,
*arguments, **named
):
"""Send signal from sender to all connected receivers catching errors
signal -- (hashable) signal value, see connect for details
sender -- the sender of the signal
if Any, only receivers registered for Any will receive
the message.
if Anonymous, only receivers registered to receive
messages from Anonymous or Any will receive the message
Otherwise can be any python object (normally one
registered with a connect if you actually want
something to occur).
arguments -- positional arguments which will be passed to
*all* receivers. Note that this may raise TypeErrors
if the receivers do not allow the particular arguments.
Note also that arguments are applied before named
arguments, so they should be used with care.
named -- named arguments which will be filtered according
to the parameters of the receivers to only provide those
acceptable to the receiver.
Return a list of tuple pairs [(receiver, response), ... ]
if any receiver raises an error (specifically any subclass of Exception),
the error instance is returned as the result for that receiver.
"""
# Call each receiver with whatever arguments it can accept.
# Return a list of tuple pairs [(receiver, response), ... ].
responses = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
try:
response = robustApply(
receiver,
signal=signal,
sender=sender,
*arguments,
**named
)
except Exception as err:
responses.append((receiver, err))
else:
responses.append((receiver, response))
return responses
def sendRobust(signal=Any, sender=Anonymous, *arguments, **named):
"""Send signal from sender to all connected receivers catching errors
signal -- (hashable) signal value, see connect for details
sender -- the sender of the signal
if Any, only receivers registered for Any will receive
the message.
if Anonymous, only receivers registered to receive
messages from Anonymous or Any will receive the message
Otherwise can be any python object (normally one
registered with a connect if you actually want
something to occur).
arguments -- positional arguments which will be passed to
*all* receivers. Note that this may raise TypeErrors
if the receivers do not allow the particular arguments.
Note also that arguments are applied before named
arguments, so they should be used with care.
named -- named arguments which will be filtered according
to the parameters of the receivers to only provide those
acceptable to the receiver.
Return a list of tuple pairs [(receiver, response), ... ]
if any receiver raises an error (specifically any subclass of Exception),
the error instance is returned as the result for that receiver.
"""
# Call each receiver with whatever arguments it can accept.
# Return a list of tuple pairs [(receiver, response), ... ].
responses = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
try:
response = robustApply(receiver,
signal=signal,
sender=sender,
*arguments,
**named)
except Exception as err:
responses.append((receiver, err))
else:
responses.append((receiver, response))
return responses
def show_receivers(self, sender=dispatcher.Any, signal=dispatcher.Any):
print("*** live receivers ***")
for rec in dispatcher.liveReceivers(dispatcher.getReceivers(sender, signal)):
print("receiver -->", rec)
print("*** end live receivers ***")
def disconnect_all(signal=Any, sender=Any):
"""Disconnect all signal handlers. Useful for cleaning up after running
tests
"""
for receiver in liveReceivers(getAllReceivers(sender, signal)):
disconnect(receiver, signal=signal, sender=sender)
def _update_channel_variables(self):
for variable in self._channel.get_variables():
# TODO: it may be resource expensive
variable.enabled = bool(liveReceivers(getAllReceivers(sender=variable)))
def disconnect_all(signal=Any, sender=Any):
"""Disconnect all signal handlers. Useful for cleaning up after running
tests
"""
for receiver in liveReceivers(getAllReceivers(sender, signal)):
disconnect(receiver, signal=signal, sender=sender)
def show_receivers(self, sender=dispatcher.Any, signal=dispatcher.Any):
print("*** live receivers ***")
for rec in dispatcher.liveReceivers(
dispatcher.getReceivers(sender, signal)):
print("receiver -->", rec)
print("*** end live receivers ***")
