def cancel_futures(futures):
dlist = []
for future in futures:
if not future.done() and not future.cancelled():
future.cancel()
dlist.append(maybeDeferred_coro(lambda: future))
return defer.DeferredList(dlist)
def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
"""Like send_catch_log but supports returning deferreds on signal handlers.
Returns a deferred that gets fired once all signal handlers deferreds were
fired.
"""
def logerror(failure, recv):
if dont_log is None or not isinstance(failure.value, dont_log):
logger.error("Error caught on signal handler: %(receiver)s",
{'receiver': recv},
exc_info=failure_to_exc_info(failure),
extra={'spider': spider})
return failure
dont_log = named.pop('dont_log', None)
spider = named.get('spider', None)
dfds = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
d = maybeDeferred_coro(robustApply,
receiver,
signal=signal,
sender=sender,
*arguments,
**named)
d.addErrback(logerror, receiver)
d.addBoth(lambda result: (receiver, result))
dfds.append(d)
d = DeferredList(dfds)
d.addCallback(lambda out: [x[1] for x in out])
return d
def getHostByName(self, name, timeout=()):
if name in dnscache:
result = dnscache[name]
logger.debug(f"using cache {name} {result}")
return defer.succeed(result)
timeout = sum(timeout) if timeout else self.timeout
d = maybeDeferred_coro(self._getHostByName, name, timeout)
return d
def build_instances(plan: andi.Plan, providers):
"""Build the instances dict from a plan."""
instances = {}
for cls, kwargs_spec in plan:
if cls in providers:
instances[cls] = yield maybeDeferred_coro(providers[cls])
else:
instances[cls] = cls(**kwargs_spec.kwargs(instances))
raise returnValue(instances)
async def _call():
if self.count < self.size:
self.count += 1
try:
await self.pool.put(self.count)
except asyncio.CancelledError:
self.count -= 1
raise
try:
q = await self.pool.get()
except asyncio.CancelledError:
raise
try:
await self.paused.wait()
r = await as_future(maybeDeferred_coro(f, *args, **kwargs))
finally:
self.pool.task_done()
await asyncio.shield(self.pool.put(q))
return r
def build_instances_from_providers(self, request: Request,
response: Response, plan: andi.Plan):
"""Build dependencies handled by registered providers"""
instances: Dict[Callable, Any] = {}
scrapy_provided_dependencies = self.available_dependencies_for_providers(
request, response)
dependencies_set = {cls for cls, _ in plan.dependencies}
for provider in self.providers:
provided_classes = {
cls
for cls in dependencies_set if provider.is_provided(cls)
}
provided_classes -= instances.keys(
) # ignore already provided types
if not provided_classes:
continue
kwargs = andi.plan(
provider,
is_injectable=is_injectable,
externally_provided=scrapy_provided_dependencies,
full_final_kwargs=False,
).final_kwargs(scrapy_provided_dependencies)
objs = yield maybeDeferred_coro(provider, set(provided_classes),
**kwargs)
objs_by_type: Dict[Callable,
Any] = {type(obj): obj
for obj in objs}
extra_classes = objs_by_type.keys() - provided_classes
if extra_classes:
raise UndeclaredProvidedTypeError(
f"{provider} has returned instances of types {extra_classes} "
"that are not among the declared supported classes in the "
f"provider: {provider.provided_classes}")
instances.update(objs_by_type)
return instances
def fn(*args, **kwargs):
yield maybeDeferred_coro(f, *args, **kwargs)
def build_instances_from_providers(self, request: Request,
response: Response, plan: andi.Plan):
"""Build dependencies handled by registered providers"""
instances: Dict[Callable, Any] = {}
scrapy_provided_dependencies = self.available_dependencies_for_providers(
request, response)
dependencies_set = {cls for cls, _ in plan.dependencies}
for provider in self.providers:
provided_classes = {
cls
for cls in dependencies_set if provider.is_provided(cls)
}
provided_classes -= instances.keys(
) # ignore already provided types
if not provided_classes:
continue
objs, fingerprint = None, None
cache_hit = False
if self.cache and provider.has_cache_support:
if not provider.name:
raise NotImplementedError(
f"The provider {type(provider)} must have a `name` defined if"
f" you want to use the cache. It must be unique across the providers."
)
# Return the data if it is already in the cache
fingerprint = f"{provider.name}_{provider.fingerprint(set(provided_classes), request)}"
try:
data = self.cache[fingerprint]
except KeyError:
self.crawler.stats.inc_value("scrapy-poet/cache/miss")
else:
self.crawler.stats.inc_value("scrapy-poet/cache/hit")
if isinstance(data, Exception):
raise data
objs = provider.deserialize(data)
cache_hit = True
if not objs:
kwargs = andi.plan(
provider,
is_injectable=is_injectable,
externally_provided=scrapy_provided_dependencies,
full_final_kwargs=False,
).final_kwargs(scrapy_provided_dependencies)
try:
# Invoke the provider to get the data
objs = yield maybeDeferred_coro(provider,
set(provided_classes),
**kwargs)
except Exception as e:
if self.cache and self.caching_errors and provider.has_cache_support:
# Save errors in the cache
self.cache[fingerprint] = e
self.crawler.stats.inc_value(
"scrapy-poet/cache/firsthand")
raise
objs_by_type: Dict[Callable,
Any] = {type(obj): obj
for obj in objs}
extra_classes = objs_by_type.keys() - provided_classes
if extra_classes:
raise UndeclaredProvidedTypeError(
f"{provider} has returned instances of types {extra_classes} "
"that are not among the declared supported classes in the "
f"provider: {provider.provided_classes}")
instances.update(objs_by_type)
if self.cache and not cache_hit and provider.has_cache_support:
# Save the results in the cache
self.cache[fingerprint] = provider.serialize(objs)
self.crawler.stats.inc_value("scrapy-poet/cache/firsthand")
return instances
def maybeDeferred(self, f, *args, **kwargs):
return maybeDeferred_coro(self.maybeFuture(f, *args, **kwargs))