def build_plan(callback, response) -> Tuple[andi.Plan, Dict[Type, Callable]]: """ Build a plan for the injection in the callback """ provider_instances = build_providers(response) plan = andi.plan(callback, is_injectable=is_injectable, externally_provided=provider_instances.keys()) return plan, provider_instances
def test_plan_callable_object(): class MyFunc: def __call__(self, b: B): pass func = MyFunc() plan = andi.plan(func, is_injectable={B}) assert plan == [(B, {}), (func, {'b': B})]
def build_plan(self, request: Request) -> andi.Plan: """Create a plan for building the dependencies required by the callback""" callback = get_callback(request, self.spider) return andi.plan( callback, is_injectable=is_injectable, externally_provided=self.is_class_provided_by_any_provider, overrides=self.overrides_registry.overrides_for(request).get)
def test_cyclic_dependency(): plan = andi.plan(E, is_injectable=lambda x: True, externally_provided={A}) # No error if externally provided with pytest.raises(andi.NonProvidableError) as exec_info: andi.plan(E, is_injectable=lambda x: True, externally_provided=[]) expected_errors = [ ('c', [(CyclicDependencyErrCase(E, [E, C, A]))]), ('d', [ (CyclicDependencyErrCase(E, [E, D, A])), (CyclicDependencyErrCase(E, [E, D, C, A])), ]) ] assert error_causes(exec_info) == expected_errors with pytest.raises(andi.NonProvidableError) as exec_info: andi.plan(E, is_injectable=lambda x: True, externally_provided=[], full_final_kwargs=True) assert error_causes(exec_info) == expected_errors
def build_plan(callback, instances) -> Tuple[andi.Plan, Dict[Type, PageObjectInputProvider]]: """Build a plan for the injection in the callback.""" provider_instances = build_providers(instances) plan = andi.plan( callback, is_injectable=is_injectable, externally_provided=provider_instances.keys() ) return plan, provider_instances
def test_plan_non_annotated_args(): class WithNonAnnArgs: def __init__(self, a: A, b: B, non_ann, non_ann_def=0, *, non_ann_kw, non_ann_kw_def=1): pass plan = andi.plan( WithNonAnnArgs.__init__, is_injectable=ALL, externally_provided={A} ) assert dict(plan.dependencies) == {A: {}, B: {}} assert _final_kwargs_spec(plan) == {'a': A, 'b': B} assert not plan.full_final_kwargs plan_class = andi.plan(WithNonAnnArgs, is_injectable=ALL, externally_provided=[A]) assert plan_class.dependencies == plan.dependencies assert _final_kwargs_spec(plan_class) == _final_kwargs_spec(plan) assert not plan.full_final_kwargs with pytest.raises(TypeError): build(plan) instances = build(plan.dependencies, instances_stock={A: ""}) o = WithNonAnnArgs(non_ann=None, non_ann_kw=None, **plan.final_kwargs(instances)) assert isinstance(o, WithNonAnnArgs) with pytest.raises(andi.NonProvidableError) as ex_info: andi.plan(WithNonAnnArgs, is_injectable=ALL, externally_provided=[A], full_final_kwargs=True) assert error_causes(ex_info) == [ ('non_ann', [LackingAnnotationErrCase('non_ann', WithNonAnnArgs)]), ('non_ann_def', [LackingAnnotationErrCase('non_ann_def', WithNonAnnArgs)]), ('non_ann_kw', [LackingAnnotationErrCase('non_ann_kw', WithNonAnnArgs)]), ('non_ann_kw_def', [LackingAnnotationErrCase('non_ann_kw_def', WithNonAnnArgs)]), ]
def test_plan_no_args(full_final_kwargs): def fn(): return True plan = andi.plan(fn, is_injectable=[], full_final_kwargs=full_final_kwargs) assert plan == [(fn, {})] assert plan.full_final_kwargs instances = build(plan) assert instances[fn] assert fn(**plan.final_kwargs(instances))
def test_plan_with_optionals_and_union(): def fn(str_or_b_or_None: Optional[Union[str, B]]): return str_or_b_or_None plan = andi.plan(fn, is_injectable={str, B, type(None)}) assert type(build(plan)[fn]) == str plan = andi.plan(fn, is_injectable={B, type(None)}) assert type(build(plan)[fn]) == B plan = andi.plan(fn, is_injectable={B, type(None)}, externally_provided={str}) assert type(build(plan)[fn]) == str plan = andi.plan(fn, is_injectable={type(None)}) assert build(plan)[fn] is None plan = andi.plan(fn, is_injectable={type(None)}, externally_provided={str}) assert type(build(plan)[fn]) == str plan = andi.plan(fn, is_injectable={type(None)}, externally_provided={str, B}) assert type(build(plan)[fn]) == str plan = andi.plan(fn, is_injectable={type(None)}, externally_provided={B}) assert type(build(plan)[fn]) == B plan = andi.plan(fn, is_injectable={}) assert plan == [(fn, {})] assert not plan.full_final_kwargs with pytest.raises(NonProvidableError) as ex_info: andi.plan(fn, is_injectable={}, full_final_kwargs=True) assert error_causes(ex_info) == [ ('str_or_b_or_None', [ NonInjectableOrExternalErrCase('str_or_b_or_None', fn, [str, B, type(None)]) ]) ]
def is_provider_requiring_scrapy_response(provider): """Check whether injectable provider makes use of a valid Response.""" plan = andi.plan( provider.__call__, is_injectable=is_injectable, externally_provided=SCRAPY_PROVIDED_CLASSES, ) for possible_type, _ in plan.dependencies: if issubclass(possible_type, Response): return True return False
def is_provider_using_response(provider): """Check whether injectable provider makes use of a valid Response.""" plan = andi.plan( provider, is_injectable=is_injectable, externally_provided=_SCRAPY_PROVIDED_CLASSES, ) for possible_type, _ in plan: if issubclass(possible_type, Response): return True return False
def build_providers(instances) -> Dict[Type, PageObjectInputProvider]: result = {} for cls, provider in providers.items(): kwargs = andi.plan( provider, is_injectable=is_injectable, externally_provided=instances.keys(), full_final_kwargs=True, ).final_kwargs(instances) result[cls] = provider(**kwargs) # type: ignore return result
def discover_callback_providers(callback): plan = andi.plan( callback, is_injectable=is_injectable, externally_provided=providers.keys(), ) for obj, _ in plan: provider = providers.get(obj) if not provider: continue yield provider
def test_plan_similar_for_class_or_func(cls, is_injectable, externally_provided): is_injectable = is_injectable + [cl.__init__ for cl in is_injectable] externally_provided = externally_provided + [cl.__init__ for cl in externally_provided] external_deps = {cl: "external" for cl in externally_provided} plan_cls = andi.plan(cls, is_injectable=is_injectable, externally_provided=externally_provided) plan_func = andi.plan(cls.__init__, is_injectable=is_injectable, externally_provided=externally_provided) plan_func[-1] = (cls, plan_func[-1][1]) # To make plans compatible assert plan_cls == plan_func assert plan_cls.full_final_kwargs assert plan_cls.full_final_kwargs == plan_func.full_final_kwargs instances = build(plan_cls, external_deps) assert type(instances[cls]) == cls or instances[cls] == "external" instances = build(plan_func, external_deps) assert type(instances[cls]) == cls or instances[cls] == "external"
def test_plan_for_func(): def fn(other: str, e: E, c: C): assert other == 'yeah!' assert type(e) == E assert type(c) == C plan = andi.plan(fn, is_injectable=ALL, externally_provided={A}) assert _final_kwargs_spec(plan) == {'e': E, 'c': C} assert not plan.full_final_kwargs instances = build(plan.dependencies, {A: ""}) fn(other="yeah!", **plan.final_kwargs(instances)) with pytest.raises(TypeError): build(plan, {A: ""}) with pytest.raises(andi.NonProvidableError) as ex_info: andi.plan(fn, is_injectable=ALL, externally_provided=[A], full_final_kwargs=True) assert error_causes(ex_info) == [ ('other', [NonInjectableOrExternalErrCase('other', fn, [str])])]
def test_cannot_be_provided(): class WithC: def __init__(self, c: C): pass plan = andi.plan(WithC, is_injectable={B, C}, externally_provided={A}) assert dict(plan) == {A: {}, B: {}, C: {'a': A, 'b': B}, WithC: {'c': C}} assert plan.full_final_kwargs # partial plan also allowed (C is not required to be injectable): plan = andi.plan(WithC, is_injectable={B}, externally_provided={A}) assert not plan.full_final_kwargs # But should fail on full_final_kwargs regimen with pytest.raises(andi.NonProvidableError) as ex_info: andi.plan(WithC, is_injectable={B}, externally_provided={A}, full_final_kwargs=True) assert error_causes(ex_info) == [('c', [ NonInjectableOrExternalErrCase('c', WithC, [C])],)] # C is injectable, but A and B are not injectable. So an exception is raised: # every single injectable dependency found must be satisfiable. with pytest.raises(andi.NonProvidableError) as ex_info: andi.plan(WithC, is_injectable=[C], full_final_kwargs=True) assert error_causes(ex_info) == [ ('c', [ NonInjectableOrExternalErrCase('a', C, [A]), NonInjectableOrExternalErrCase('b', C, [B]), ]), ]
def test_externally_provided(): plan = andi.plan(E.__init__, is_injectable=ALL, externally_provided=ALL) assert dict(plan.dependencies) == {B: {}, C: {}, D: {}} assert _final_kwargs_spec(plan) == {'b': B, 'c': C, 'd': D} assert plan.full_final_kwargs plan = andi.plan(E.__init__, is_injectable=[], externally_provided=ALL) assert dict(plan.dependencies) == {B: {}, C: {}, D: {}} assert _final_kwargs_spec(plan) == {'b': B, 'c': C, 'd': D} assert plan.full_final_kwargs plan = andi.plan(E, is_injectable=ALL, externally_provided=ALL) assert plan == [(E, {})] assert _final_kwargs_spec(plan) == {} assert plan.dependencies == [] assert plan.full_final_kwargs plan = andi.plan(E, is_injectable=ALL, externally_provided={A, B, C, D}) assert dict(plan).keys() == {B, C, D, E} assert plan[-1][0] == E assert _final_kwargs_spec(plan) == {'b': B, 'c': C, 'd': D} assert _final_kwargs_spec(plan) == plan[-1][1] assert plan.full_final_kwargs plan = andi.plan(E, is_injectable=ALL, externally_provided={A, B, D}) plan_od = OrderedDict(plan) seq = list(plan_od.keys()) assert seq.index(A) < seq.index(C) assert seq.index(B) < seq.index(C) assert seq.index(D) < seq.index(E) assert seq.index(C) < seq.index(E) for cls in (A, B, D): assert plan_od[cls] == {} assert plan_od[C] == {'a': A, 'b': B} assert plan_od[E] == {'b': B, 'c': C, 'd': D} assert plan.full_final_kwargs
def test_plan_and_build(): plan = andi.plan(E, is_injectable=lambda x: True, externally_provided={A}) assert dict(plan[:2]).keys() == {A, B} assert list(dict(plan[:2]).values()) == [{}, {}] assert plan[2:] == [ (C, {'a': A, 'b': B}), (D, {'a': A, 'c': C}), (E, {'b': B, 'c': C, 'd': D}) ] assert plan.full_final_kwargs instances = build(plan, {A: ""}) assert type(instances[E]) == E
def test_plan_use_fn_as_annotations(full_final_kwargs): def fn_ann(b: B): setattr(b, "modified", True) return b def fn(b: fn_ann): return b plan = andi.plan(fn, is_injectable=[fn_ann, B], full_final_kwargs=full_final_kwargs) assert plan.full_final_kwargs instances = build(plan) assert instances[fn].modified
def test_plan_with_optionals(): def fn(a: Optional[str]): assert a is None return "invoked!" plan = andi.plan(fn, is_injectable={type(None), str}, externally_provided={str}) assert plan == [(str, {}), (fn, {'a': str})] assert plan.full_final_kwargs plan = andi.plan(fn, is_injectable={type(None)}) assert plan.dependencies == [(type(None), {})] assert _final_kwargs_spec(plan) == {'a': type(None)} assert plan.full_final_kwargs instances = build(plan) assert instances[type(None)] is None assert instances[fn] == "invoked!" with pytest.raises(andi.NonProvidableError) as ex_info: andi.plan(fn, is_injectable={}, full_final_kwargs=True) assert error_causes(ex_info) == [ ('a', [NonInjectableOrExternalErrCase('a', fn, [str, type(None)])])]
def test_plan_with_union(): class WithUnion: def __init__(self, a_or_b: Union[A, B]): pass plan = andi.plan(WithUnion, is_injectable={WithUnion, A, B}, externally_provided={A}) assert plan == [(A, {}), (WithUnion, {'a_or_b': A})] assert plan.full_final_kwargs plan = andi.plan(WithUnion, is_injectable={WithUnion, B}, externally_provided={A}) assert plan == [(A, {}), (WithUnion, {'a_or_b': A})] assert plan.full_final_kwargs plan = andi.plan(WithUnion, is_injectable={WithUnion, B}) assert plan == [(B, {}), (WithUnion, {'a_or_b': B})] assert plan.full_final_kwargs plan = andi.plan(WithUnion, is_injectable={WithUnion}, externally_provided={B}) assert plan == [(B, {}), (WithUnion, {'a_or_b': B})] assert plan.full_final_kwargs with pytest.raises(andi.NonProvidableError) as ex_info: andi.plan(WithUnion, is_injectable={WithUnion}, full_final_kwargs=True) assert error_causes(ex_info) == [ ('a_or_b', [NonInjectableOrExternalErrCase('a_or_b', WithUnion, [A, B])]) ] with pytest.raises(andi.NonProvidableError) as ex_info: andi.plan(WithUnion, is_injectable={}, full_final_kwargs=True) assert error_causes(ex_info) == [ ('a_or_b', [NonInjectableOrExternalErrCase('a_or_b', WithUnion, [A, B])]) ]
def build_instances_from_providers(self, request: Request, response: Response, plan: andi.Plan): """Build dependencies handled by registered providers""" instances: Dict[Callable, Any] = {} scrapy_provided_dependencies = self.available_dependencies_for_providers( request, response) dependencies_set = {cls for cls, _ in plan.dependencies} for provider in self.providers: provided_classes = { cls for cls in dependencies_set if provider.is_provided(cls) } provided_classes -= instances.keys( ) # ignore already provided types if not provided_classes: continue kwargs = andi.plan( provider, is_injectable=is_injectable, externally_provided=scrapy_provided_dependencies, full_final_kwargs=False, ).final_kwargs(scrapy_provided_dependencies) objs = yield maybeDeferred_coro(provider, set(provided_classes), **kwargs) objs_by_type: Dict[Callable, Any] = {type(obj): obj for obj in objs} extra_classes = objs_by_type.keys() - provided_classes if extra_classes: raise UndeclaredProvidedTypeError( f"{provider} has returned instances of types {extra_classes} " "that are not among the declared supported classes in the " f"provider: {provider.provided_classes}") instances.update(objs_by_type) return instances
def build_instances_from_providers(self, request: Request, response: Response, plan: andi.Plan): """Build dependencies handled by registered providers""" instances: Dict[Callable, Any] = {} scrapy_provided_dependencies = self.available_dependencies_for_providers( request, response) dependencies_set = {cls for cls, _ in plan.dependencies} for provider in self.providers: provided_classes = { cls for cls in dependencies_set if provider.is_provided(cls) } provided_classes -= instances.keys( ) # ignore already provided types if not provided_classes: continue objs, fingerprint = None, None cache_hit = False if self.cache and provider.has_cache_support: if not provider.name: raise NotImplementedError( f"The provider {type(provider)} must have a `name` defined if" f" you want to use the cache. It must be unique across the providers." ) # Return the data if it is already in the cache fingerprint = f"{provider.name}_{provider.fingerprint(set(provided_classes), request)}" try: data = self.cache[fingerprint] except KeyError: self.crawler.stats.inc_value("scrapy-poet/cache/miss") else: self.crawler.stats.inc_value("scrapy-poet/cache/hit") if isinstance(data, Exception): raise data objs = provider.deserialize(data) cache_hit = True if not objs: kwargs = andi.plan( provider, is_injectable=is_injectable, externally_provided=scrapy_provided_dependencies, full_final_kwargs=False, ).final_kwargs(scrapy_provided_dependencies) try: # Invoke the provider to get the data objs = yield maybeDeferred_coro(provider, set(provided_classes), **kwargs) except Exception as e: if self.cache and self.caching_errors and provider.has_cache_support: # Save errors in the cache self.cache[fingerprint] = e self.crawler.stats.inc_value( "scrapy-poet/cache/firsthand") raise objs_by_type: Dict[Callable, Any] = {type(obj): obj for obj in objs} extra_classes = objs_by_type.keys() - provided_classes if extra_classes: raise UndeclaredProvidedTypeError( f"{provider} has returned instances of types {extra_classes} " "that are not among the declared supported classes in the " f"provider: {provider.provided_classes}") instances.update(objs_by_type) if self.cache and not cache_hit and provider.has_cache_support: # Save the results in the cache self.cache[fingerprint] = provider.serialize(objs) self.crawler.stats.inc_value("scrapy-poet/cache/firsthand") return instances