def build_plan(callback, response) -> Tuple[andi.Plan, Dict[Type, Callable]]:
""" Build a plan for the injection in the callback """
provider_instances = build_providers(response)
plan = andi.plan(callback,
is_injectable=is_injectable,
externally_provided=provider_instances.keys())
return plan, provider_instances
def test_plan_callable_object():
class MyFunc:
def __call__(self, b: B):
pass
func = MyFunc()
plan = andi.plan(func, is_injectable={B})
assert plan == [(B, {}), (func, {'b': B})]
def build_plan(self, request: Request) -> andi.Plan:
"""Create a plan for building the dependencies required by the callback"""
callback = get_callback(request, self.spider)
return andi.plan(
callback,
is_injectable=is_injectable,
externally_provided=self.is_class_provided_by_any_provider,
overrides=self.overrides_registry.overrides_for(request).get)
def test_cyclic_dependency():
plan = andi.plan(E, is_injectable=lambda x: True,
externally_provided={A}) # No error if externally provided
with pytest.raises(andi.NonProvidableError) as exec_info:
andi.plan(E, is_injectable=lambda x: True, externally_provided=[])
expected_errors = [
('c', [(CyclicDependencyErrCase(E, [E, C, A]))]),
('d', [
(CyclicDependencyErrCase(E, [E, D, A])),
(CyclicDependencyErrCase(E, [E, D, C, A])),
])
]
assert error_causes(exec_info) == expected_errors
with pytest.raises(andi.NonProvidableError) as exec_info:
andi.plan(E, is_injectable=lambda x: True, externally_provided=[],
full_final_kwargs=True)
assert error_causes(exec_info) == expected_errors
def build_plan(callback, instances) -> Tuple[andi.Plan, Dict[Type, PageObjectInputProvider]]:
"""Build a plan for the injection in the callback."""
provider_instances = build_providers(instances)
plan = andi.plan(
callback,
is_injectable=is_injectable,
externally_provided=provider_instances.keys()
)
return plan, provider_instances
def test_plan_non_annotated_args():
class WithNonAnnArgs:
def __init__(self, a: A, b: B, non_ann, non_ann_def=0, *,
non_ann_kw, non_ann_kw_def=1):
pass
plan = andi.plan(
WithNonAnnArgs.__init__,
is_injectable=ALL,
externally_provided={A}
)
assert dict(plan.dependencies) == {A: {}, B: {}}
assert _final_kwargs_spec(plan) == {'a': A, 'b': B}
assert not plan.full_final_kwargs
plan_class = andi.plan(WithNonAnnArgs,
is_injectable=ALL,
externally_provided=[A])
assert plan_class.dependencies == plan.dependencies
assert _final_kwargs_spec(plan_class) == _final_kwargs_spec(plan)
assert not plan.full_final_kwargs
with pytest.raises(TypeError):
build(plan)
instances = build(plan.dependencies, instances_stock={A: ""})
o = WithNonAnnArgs(non_ann=None, non_ann_kw=None,
**plan.final_kwargs(instances))
assert isinstance(o, WithNonAnnArgs)
with pytest.raises(andi.NonProvidableError) as ex_info:
andi.plan(WithNonAnnArgs, is_injectable=ALL,
externally_provided=[A], full_final_kwargs=True)
assert error_causes(ex_info) == [
('non_ann', [LackingAnnotationErrCase('non_ann', WithNonAnnArgs)]),
('non_ann_def', [LackingAnnotationErrCase('non_ann_def',
WithNonAnnArgs)]),
('non_ann_kw', [LackingAnnotationErrCase('non_ann_kw',
WithNonAnnArgs)]),
('non_ann_kw_def', [LackingAnnotationErrCase('non_ann_kw_def',
WithNonAnnArgs)]),
]
def test_plan_no_args(full_final_kwargs):
def fn():
return True
plan = andi.plan(fn, is_injectable=[], full_final_kwargs=full_final_kwargs)
assert plan == [(fn, {})]
assert plan.full_final_kwargs
instances = build(plan)
assert instances[fn]
assert fn(**plan.final_kwargs(instances))
def test_plan_with_optionals_and_union():
def fn(str_or_b_or_None: Optional[Union[str, B]]):
return str_or_b_or_None
plan = andi.plan(fn, is_injectable={str, B, type(None)})
assert type(build(plan)[fn]) == str
plan = andi.plan(fn, is_injectable={B, type(None)})
assert type(build(plan)[fn]) == B
plan = andi.plan(fn, is_injectable={B, type(None)},
externally_provided={str})
assert type(build(plan)[fn]) == str
plan = andi.plan(fn, is_injectable={type(None)})
assert build(plan)[fn] is None
plan = andi.plan(fn, is_injectable={type(None)},
externally_provided={str})
assert type(build(plan)[fn]) == str
plan = andi.plan(fn, is_injectable={type(None)},
externally_provided={str, B})
assert type(build(plan)[fn]) == str
plan = andi.plan(fn, is_injectable={type(None)},
externally_provided={B})
assert type(build(plan)[fn]) == B
plan = andi.plan(fn, is_injectable={})
assert plan == [(fn, {})]
assert not plan.full_final_kwargs
with pytest.raises(NonProvidableError) as ex_info:
andi.plan(fn, is_injectable={}, full_final_kwargs=True)
assert error_causes(ex_info) == [
('str_or_b_or_None', [
NonInjectableOrExternalErrCase('str_or_b_or_None', fn,
[str, B, type(None)])
])
]
def is_provider_requiring_scrapy_response(provider):
"""Check whether injectable provider makes use of a valid Response."""
plan = andi.plan(
provider.__call__,
is_injectable=is_injectable,
externally_provided=SCRAPY_PROVIDED_CLASSES,
)
for possible_type, _ in plan.dependencies:
if issubclass(possible_type, Response):
return True
return False
def is_provider_using_response(provider):
"""Check whether injectable provider makes use of a valid Response."""
plan = andi.plan(
provider,
is_injectable=is_injectable,
externally_provided=_SCRAPY_PROVIDED_CLASSES,
)
for possible_type, _ in plan:
if issubclass(possible_type, Response):
return True
return False
def build_providers(instances) -> Dict[Type, PageObjectInputProvider]:
result = {}
for cls, provider in providers.items():
kwargs = andi.plan(
provider,
is_injectable=is_injectable,
externally_provided=instances.keys(),
full_final_kwargs=True,
).final_kwargs(instances)
result[cls] = provider(**kwargs) # type: ignore
return result
def discover_callback_providers(callback):
plan = andi.plan(
callback,
is_injectable=is_injectable,
externally_provided=providers.keys(),
)
for obj, _ in plan:
provider = providers.get(obj)
if not provider:
continue
yield provider
def test_plan_similar_for_class_or_func(cls, is_injectable, externally_provided):
is_injectable = is_injectable + [cl.__init__ for cl in is_injectable]
externally_provided = externally_provided + [cl.__init__
for cl in externally_provided]
external_deps = {cl: "external" for cl in externally_provided}
plan_cls = andi.plan(cls, is_injectable=is_injectable,
externally_provided=externally_provided)
plan_func = andi.plan(cls.__init__, is_injectable=is_injectable,
externally_provided=externally_provided)
plan_func[-1] = (cls, plan_func[-1][1]) # To make plans compatible
assert plan_cls == plan_func
assert plan_cls.full_final_kwargs
assert plan_cls.full_final_kwargs == plan_func.full_final_kwargs
instances = build(plan_cls, external_deps)
assert type(instances[cls]) == cls or instances[cls] == "external"
instances = build(plan_func, external_deps)
assert type(instances[cls]) == cls or instances[cls] == "external"
def test_plan_for_func():
def fn(other: str, e: E, c: C):
assert other == 'yeah!'
assert type(e) == E
assert type(c) == C
plan = andi.plan(fn, is_injectable=ALL,
externally_provided={A})
assert _final_kwargs_spec(plan) == {'e': E, 'c': C}
assert not plan.full_final_kwargs
instances = build(plan.dependencies, {A: ""})
fn(other="yeah!", **plan.final_kwargs(instances))
with pytest.raises(TypeError):
build(plan, {A: ""})
with pytest.raises(andi.NonProvidableError) as ex_info:
andi.plan(fn, is_injectable=ALL, externally_provided=[A],
full_final_kwargs=True)
assert error_causes(ex_info) == [
('other', [NonInjectableOrExternalErrCase('other', fn, [str])])]
def test_cannot_be_provided():
class WithC:
def __init__(self, c: C):
pass
plan = andi.plan(WithC, is_injectable={B, C}, externally_provided={A})
assert dict(plan) == {A: {}, B: {}, C: {'a': A, 'b': B}, WithC: {'c': C}}
assert plan.full_final_kwargs
# partial plan also allowed (C is not required to be injectable):
plan = andi.plan(WithC, is_injectable={B}, externally_provided={A})
assert not plan.full_final_kwargs
# But should fail on full_final_kwargs regimen
with pytest.raises(andi.NonProvidableError) as ex_info:
andi.plan(WithC, is_injectable={B}, externally_provided={A},
full_final_kwargs=True)
assert error_causes(ex_info) == [('c', [
NonInjectableOrExternalErrCase('c', WithC, [C])],)]
# C is injectable, but A and B are not injectable. So an exception is raised:
# every single injectable dependency found must be satisfiable.
with pytest.raises(andi.NonProvidableError) as ex_info:
andi.plan(WithC, is_injectable=[C], full_final_kwargs=True)
assert error_causes(ex_info) == [
('c', [
NonInjectableOrExternalErrCase('a', C, [A]),
NonInjectableOrExternalErrCase('b', C, [B]),
]),
]
def test_externally_provided():
plan = andi.plan(E.__init__, is_injectable=ALL,
externally_provided=ALL)
assert dict(plan.dependencies) == {B: {}, C: {}, D: {}}
assert _final_kwargs_spec(plan) == {'b': B, 'c': C, 'd': D}
assert plan.full_final_kwargs
plan = andi.plan(E.__init__, is_injectable=[],
externally_provided=ALL)
assert dict(plan.dependencies) == {B: {}, C: {}, D: {}}
assert _final_kwargs_spec(plan) == {'b': B, 'c': C, 'd': D}
assert plan.full_final_kwargs
plan = andi.plan(E, is_injectable=ALL, externally_provided=ALL)
assert plan == [(E, {})]
assert _final_kwargs_spec(plan) == {}
assert plan.dependencies == []
assert plan.full_final_kwargs
plan = andi.plan(E, is_injectable=ALL,
externally_provided={A, B, C, D})
assert dict(plan).keys() == {B, C, D, E}
assert plan[-1][0] == E
assert _final_kwargs_spec(plan) == {'b': B, 'c': C, 'd': D}
assert _final_kwargs_spec(plan) == plan[-1][1]
assert plan.full_final_kwargs
plan = andi.plan(E, is_injectable=ALL,
externally_provided={A, B, D})
plan_od = OrderedDict(plan)
seq = list(plan_od.keys())
assert seq.index(A) < seq.index(C)
assert seq.index(B) < seq.index(C)
assert seq.index(D) < seq.index(E)
assert seq.index(C) < seq.index(E)
for cls in (A, B, D):
assert plan_od[cls] == {}
assert plan_od[C] == {'a': A, 'b': B}
assert plan_od[E] == {'b': B, 'c': C, 'd': D}
assert plan.full_final_kwargs
def test_plan_and_build():
plan = andi.plan(E, is_injectable=lambda x: True,
externally_provided={A})
assert dict(plan[:2]).keys() == {A, B}
assert list(dict(plan[:2]).values()) == [{}, {}]
assert plan[2:] == [
(C, {'a': A, 'b': B}),
(D, {'a': A, 'c': C}),
(E, {'b': B, 'c': C, 'd': D})
]
assert plan.full_final_kwargs
instances = build(plan, {A: ""})
assert type(instances[E]) == E
def test_plan_use_fn_as_annotations(full_final_kwargs):
def fn_ann(b: B):
setattr(b, "modified", True)
return b
def fn(b: fn_ann):
return b
plan = andi.plan(fn, is_injectable=[fn_ann, B],
full_final_kwargs=full_final_kwargs)
assert plan.full_final_kwargs
instances = build(plan)
assert instances[fn].modified
def test_plan_with_optionals():
def fn(a: Optional[str]):
assert a is None
return "invoked!"
plan = andi.plan(fn, is_injectable={type(None), str},
externally_provided={str})
assert plan == [(str, {}), (fn, {'a': str})]
assert plan.full_final_kwargs
plan = andi.plan(fn, is_injectable={type(None)})
assert plan.dependencies == [(type(None), {})]
assert _final_kwargs_spec(plan) == {'a': type(None)}
assert plan.full_final_kwargs
instances = build(plan)
assert instances[type(None)] is None
assert instances[fn] == "invoked!"
with pytest.raises(andi.NonProvidableError) as ex_info:
andi.plan(fn, is_injectable={}, full_final_kwargs=True)
assert error_causes(ex_info) == [
('a', [NonInjectableOrExternalErrCase('a', fn, [str, type(None)])])]
def test_plan_with_union():
class WithUnion:
def __init__(self, a_or_b: Union[A, B]):
pass
plan = andi.plan(WithUnion,
is_injectable={WithUnion, A, B},
externally_provided={A})
assert plan == [(A, {}), (WithUnion, {'a_or_b': A})]
assert plan.full_final_kwargs
plan = andi.plan(WithUnion,
is_injectable={WithUnion, B},
externally_provided={A})
assert plan == [(A, {}), (WithUnion, {'a_or_b': A})]
assert plan.full_final_kwargs
plan = andi.plan(WithUnion, is_injectable={WithUnion, B})
assert plan == [(B, {}), (WithUnion, {'a_or_b': B})]
assert plan.full_final_kwargs
plan = andi.plan(WithUnion, is_injectable={WithUnion},
externally_provided={B})
assert plan == [(B, {}), (WithUnion, {'a_or_b': B})]
assert plan.full_final_kwargs
with pytest.raises(andi.NonProvidableError) as ex_info:
andi.plan(WithUnion, is_injectable={WithUnion},
full_final_kwargs=True)
assert error_causes(ex_info) == [
('a_or_b',
[NonInjectableOrExternalErrCase('a_or_b', WithUnion, [A, B])])
]
with pytest.raises(andi.NonProvidableError) as ex_info:
andi.plan(WithUnion, is_injectable={}, full_final_kwargs=True)
assert error_causes(ex_info) == [
('a_or_b',
[NonInjectableOrExternalErrCase('a_or_b', WithUnion, [A, B])])
]
def build_instances_from_providers(self, request: Request,
response: Response, plan: andi.Plan):
"""Build dependencies handled by registered providers"""
instances: Dict[Callable, Any] = {}
scrapy_provided_dependencies = self.available_dependencies_for_providers(
request, response)
dependencies_set = {cls for cls, _ in plan.dependencies}
for provider in self.providers:
provided_classes = {
cls
for cls in dependencies_set if provider.is_provided(cls)
}
provided_classes -= instances.keys(
) # ignore already provided types
if not provided_classes:
continue
kwargs = andi.plan(
provider,
is_injectable=is_injectable,
externally_provided=scrapy_provided_dependencies,
full_final_kwargs=False,
).final_kwargs(scrapy_provided_dependencies)
objs = yield maybeDeferred_coro(provider, set(provided_classes),
**kwargs)
objs_by_type: Dict[Callable,
Any] = {type(obj): obj
for obj in objs}
extra_classes = objs_by_type.keys() - provided_classes
if extra_classes:
raise UndeclaredProvidedTypeError(
f"{provider} has returned instances of types {extra_classes} "
"that are not among the declared supported classes in the "
f"provider: {provider.provided_classes}")
instances.update(objs_by_type)
return instances
def build_instances_from_providers(self, request: Request,
response: Response, plan: andi.Plan):
"""Build dependencies handled by registered providers"""
instances: Dict[Callable, Any] = {}
scrapy_provided_dependencies = self.available_dependencies_for_providers(
request, response)
dependencies_set = {cls for cls, _ in plan.dependencies}
for provider in self.providers:
provided_classes = {
cls
for cls in dependencies_set if provider.is_provided(cls)
}
provided_classes -= instances.keys(
) # ignore already provided types
if not provided_classes:
continue
objs, fingerprint = None, None
cache_hit = False
if self.cache and provider.has_cache_support:
if not provider.name:
raise NotImplementedError(
f"The provider {type(provider)} must have a `name` defined if"
f" you want to use the cache. It must be unique across the providers."
)
# Return the data if it is already in the cache
fingerprint = f"{provider.name}_{provider.fingerprint(set(provided_classes), request)}"
try:
data = self.cache[fingerprint]
except KeyError:
self.crawler.stats.inc_value("scrapy-poet/cache/miss")
else:
self.crawler.stats.inc_value("scrapy-poet/cache/hit")
if isinstance(data, Exception):
raise data
objs = provider.deserialize(data)
cache_hit = True
if not objs:
kwargs = andi.plan(
provider,
is_injectable=is_injectable,
externally_provided=scrapy_provided_dependencies,
full_final_kwargs=False,
).final_kwargs(scrapy_provided_dependencies)
try:
# Invoke the provider to get the data
objs = yield maybeDeferred_coro(provider,
set(provided_classes),
**kwargs)
except Exception as e:
if self.cache and self.caching_errors and provider.has_cache_support:
# Save errors in the cache
self.cache[fingerprint] = e
self.crawler.stats.inc_value(
"scrapy-poet/cache/firsthand")
raise
objs_by_type: Dict[Callable,
Any] = {type(obj): obj
for obj in objs}
extra_classes = objs_by_type.keys() - provided_classes
if extra_classes:
raise UndeclaredProvidedTypeError(
f"{provider} has returned instances of types {extra_classes} "
"that are not among the declared supported classes in the "
f"provider: {provider.provided_classes}")
instances.update(objs_by_type)
if self.cache and not cache_hit and provider.has_cache_support:
# Save the results in the cache
self.cache[fingerprint] = provider.serialize(objs)
self.crawler.stats.inc_value("scrapy-poet/cache/firsthand")
return instances
