def test_build_instances_methods(self, injector):
def callback(response: DummyResponse, a: Cls1, b: Cls2, c: WrapCls,
d: ClsNoProviderRequired):
pass
response = get_response_for_testing(callback)
request = response.request
plan = injector.build_plan(response.request)
instances = yield from injector.build_instances(
request, response, plan)
assert instances == {
Cls1: Cls1(),
Cls2: Cls2(),
WrapCls: WrapCls(ClsReqResponse()),
ClsReqResponse: ClsReqResponse(),
ClsNoProviderRequired: ClsNoProviderRequired()
}
instances = yield from injector.build_instances_from_providers(
request, response, plan)
assert instances == {
Cls1: Cls1(),
Cls2: Cls2(),
ClsReqResponse: ClsReqResponse(),
}
def test_on_exception(self, provided_cls: AutoExtractData):
class Provider(AutoExtractProvider):
async def do_request(self, *args, agg_stats: AggStats, **kwargs):
agg_stats.n_attempts += 3
agg_stats.n_billable_query_responses += 2
raise Exception()
def callback(item: provided_cls):
pass
page_type = provided_cls.page_type
injector = get_injector_for_testing({Provider: 500})
response = get_response_for_testing(callback)
with pytest.raises(Exception) as exinf:
yield injector.build_callback_dependencies(response.request,
response)
stats = injector.crawler.stats
expected = {
f'autoextract/{page_type}/pages/count': 1,
f'autoextract/{page_type}/pages/errors': 1,
'autoextract/total/attempts/count': 3,
'autoextract/total/attempts/billable': 2,
'autoextract/total/pages/count': 1,
'autoextract/total/pages/errors': 1,
'autoextract/total/pages/errors/rest/Exception': 1
}
assert_stats(stats, expected)
def test_on_query_error(self, provided_cls: AutoExtractData):
page_type = provided_cls.page_type
data = {"query": "The query", "error": "Download error"}
class Provider(AutoExtractProvider):
async def do_request(self, *args, agg_stats: AggStats, **kwargs):
agg_stats.n_attempts += 3
agg_stats.n_billable_query_responses += 2
return [data]
def callback(item: provided_cls):
pass
injector = get_injector_for_testing({Provider: 500})
response = get_response_for_testing(callback)
with pytest.raises(QueryError) as exinf:
yield injector.build_callback_dependencies(response.request,
response)
stats = injector.crawler.stats
expected = {
f'autoextract/{page_type}/pages/count': 1,
f'autoextract/{page_type}/pages/errors': 1,
'autoextract/total/attempts/count': 3,
'autoextract/total/attempts/billable': 2,
'autoextract/total/pages/count': 1,
'autoextract/total/pages/errors': 1,
'autoextract/total/pages/errors/query/Download error': 1
}
assert_stats(stats, expected)
assert "Download error" in str(exinf.value)
assert "The query" in str(exinf.value)
def test_overrides(self, providers, override_should_happen):
domain = "example.com" if override_should_happen else "other-example.com"
# The request domain is example.com, so overrides shouldn't be applied
# when we configure them for domain other-example.com
overrides = {
domain: {
PricePO: PriceInDollarsPO,
EurDollarRate: OtherEurDollarRate
}
}
registry = PerDomainOverridesRegistry(overrides)
injector = get_injector_for_testing(providers,
overrides_registry=registry)
def callback(response: DummyResponse, price_po: PricePO, rate_po: EurDollarRate):
pass
response = get_response_for_testing(callback)
kwargs = yield from injector.build_callback_dependencies(
response.request, response)
kwargs_types = {key: type(value) for key, value in kwargs.items()}
price_po = kwargs["price_po"]
item = price_po.to_item()
if override_should_happen:
assert kwargs_types == {"price_po": PriceInDollarsPO, "rate_po": OtherEurDollarRate}
# Note that OtherEurDollarRate don't have effect inside PriceInDollarsPO
# because composability of overrides is forbidden
assert item == {"price": 22 * 1.1, "currency": "$"}
else:
assert kwargs_types == {"price_po": PricePO, "rate_po": EurDollarRate}
assert item == {"price": 22, "currency": "€"}
def test_is_scrapy_response_required(self, injector):
def callback_no_1(response: DummyResponse, a: Cls1):
pass
response = get_response_for_testing(callback_no_1)
assert not injector.is_scrapy_response_required(response.request)
def callback_yes_1(response, a: Cls1):
pass
response = get_response_for_testing(callback_yes_1)
assert injector.is_scrapy_response_required(response.request)
def callback_yes_2(response: DummyResponse, a: ClsReqResponse):
pass
response = get_response_for_testing(callback_yes_2)
assert injector.is_scrapy_response_required(response.request)
async def test_on_cancellation(self, provided_cls: AutoExtractProductData):
old_handler = signal.getsignal(SIGINT)
signal.signal(SIGINT, lambda x, y: None)
try:
lock = asyncio.Lock()
await lock.acquire()
class Provider(AutoExtractProvider):
async def do_request(self, *args, agg_stats: AggStats,
**kwargs):
await lock.acquire()
def callback(item: provided_cls):
pass
injector = get_injector_for_testing({Provider: 500})
stats = injector.crawler.stats
response = get_response_for_testing(callback)
deferred = injector.build_callback_dependencies(
response.request, response)
build_callbacks_future = Deferred.asFuture(
deferred, asyncio.get_event_loop())
async def cancel_after(sleep):
await asyncio.sleep(sleep)
pid = os.getpid()
try:
os.kill(pid, SIGINT)
except KeyboardInterrupt:
# As an effect of the SIGINT killing the process might receive
# here a KeyboardInterrupt exception. This is Ok.
pass
return CancelledError()
result = await asyncio.gather(build_callbacks_future,
cancel_after(0.05),
return_exceptions=True)
assert all([isinstance(r, CancelledError) for r in result])
page_type = provided_cls.page_type
expected_stats = {
'autoextract/total/pages/count': 1,
'autoextract/total/pages/cancelled': 1,
'autoextract/total/pages/errors': 0,
f'autoextract/{page_type}/pages/count': 1,
f'autoextract/{page_type}/pages/cancelled': 1,
f'autoextract/{page_type}/pages/errors': 0,
}
assert_stats(stats, expected_stats)
finally:
signal.signal(SIGINT, old_handler)
def test_build_instances_from_providers_respect_priorities(self, str_list):
providers = {get_provider({str}, text): int(text) for text in str_list}
injector = get_injector_for_testing(providers)
def callback(response: DummyResponse, arg: str):
pass
response = get_response_for_testing(callback)
plan = injector.build_plan(response.request)
instances = yield from injector.build_instances_from_providers(
response.request, response, plan)
assert instances[str] == min(str_list)
def test_build_callback_dependencies(self, injector):
def callback(response: DummyResponse, a: Cls1, b: Cls2, c: WrapCls,
d: ClsNoProviderRequired):
pass
response = get_response_for_testing(callback)
kwargs = yield from injector.build_callback_dependencies(
response.request, response)
kwargs_types = {key: type(value) for key, value in kwargs.items()}
assert kwargs_types == {
"a": Cls1,
"b": Cls2,
"c": WrapCls,
"d": ClsNoProviderRequired
}
def test_build_instances_from_providers_unexpected_return(self):
class WrongProvider(get_provider({Cls1})):
def __call__(self, to_provide):
return super().__call__(to_provide) + [Cls2()]
injector = get_injector_for_testing({WrongProvider: 0})
def callback(response: DummyResponse, a: Cls1):
pass
response = get_response_for_testing(callback)
plan = injector.build_plan(response.request)
with pytest.raises(UndeclaredProvidedTypeError) as exinf:
yield from injector.build_instances_from_providers(
response.request, response, plan)
assert "Provider" in str(exinf.value)
assert "Cls2" in str(exinf.value)
assert "Cls1" in str(exinf.value)
def test_providers(self, provided_cls: AutoExtractProductData):
page_type = provided_cls.page_type
url, html = "http://example.com", "html_content"
data_wo_html = {page_type: {"url": url}}
data = {page_type: {"url": url}, "html": html}
provider_wrapper = []
class Provider(AutoExtractProvider):
async def do_request(self, *args, agg_stats: AggStats, **kwargs):
assert provider.aiohttp_session.connector.limit == 2020
agg_stats.n_attempts += 3
agg_stats.n_billable_query_responses += 2
assert kwargs['api_key'] == "key"
assert kwargs['endpoint'] == "url"
assert kwargs['max_query_error_retries'] == 31415
return [copy.deepcopy(data)]
def callback(item: provided_cls):
pass
def callback_with_html(item: provided_cls, html: AutoExtractHtml):
pass
def callback_only_html(html: AutoExtractHtml):
pass
settings = {
"AUTOEXTRACT_USER": "******",
"AUTOEXTRACT_URL": "url",
"AUTOEXTRACT_MAX_QUERY_ERROR_RETRIES": 31415,
"CONCURRENT_REQUESTS": 2020,
"CONCURRENT_REQUESTS_PER_DOMAIN": 1980,
}
injector = get_injector_for_testing({Provider: 500}, settings)
stats = injector.crawler.stats
provider = injector.providers[-1]
provider_wrapper.append(provider)
assert provider.per_domain_semaphore.concurrency_per_slot == 1980
# - No HTML requested case -
response = get_response_for_testing(callback)
kwargs = yield injector.build_callback_dependencies(
response.request, response)
assert kwargs["item"].data == data_wo_html
assert type(kwargs["item"]) is provided_cls
expected_stats = {
'autoextract/total/pages/count': 1,
'autoextract/total/pages/success': 1,
'autoextract/total/attempts/count': 3,
'autoextract/total/attempts/billable': 2,
f'autoextract/{page_type}/pages/count': 1,
f'autoextract/{page_type}/pages/success': 1
}
assert_stats(stats, expected_stats)
# - Both HTML and item requested case -
response = get_response_for_testing(callback_with_html)
kwargs = yield injector.build_callback_dependencies(
response.request, response)
item, html_response = kwargs["item"], kwargs["html"]
assert item.data == data_wo_html
assert type(item) is provided_cls
assert (html_response.url, html_response.html) == (url, html)
assert type(html_response) is AutoExtractHtml
expected_stats = {
'autoextract/total/pages/count': 2,
'autoextract/total/pages/success': 2,
'autoextract/total/pages/html': 1,
'autoextract/total/attempts/count': 6,
'autoextract/total/attempts/billable': 4,
f'autoextract/{page_type}/pages/count': 2,
f'autoextract/{page_type}/pages/success': 2,
f'autoextract/{page_type}/pages/html': 1,
}
assert_stats(stats, expected_stats)
# - Only HTML is requested case -
injector.providers[0].page_type_class_for_html = provided_cls
response = get_response_for_testing(callback_only_html)
kwargs = yield injector.build_callback_dependencies(
response.request, response)
assert "item" not in kwargs
html_response = kwargs["html"]
assert (html_response.url, html_response.html) == (url, html)
assert type(html_response) is AutoExtractHtml
expected_stats = {
'autoextract/total/pages/count': 3,
'autoextract/total/pages/success': 3,
'autoextract/total/pages/html': 2,
'autoextract/total/attempts/count': 9,
'autoextract/total/attempts/billable': 6,
f'autoextract/{page_type}/pages/count': 3,
f'autoextract/{page_type}/pages/success': 3,
f'autoextract/{page_type}/pages/html': 2,
}
assert_stats(stats, expected_stats)
def test_cache(tmp_path, cache_errors):
"""
In a first run, the cache is empty, and two requests are done, one with exception.
In the second run we should get the same result as in the first run. The
behaviour for exceptions vary if caching errors is disabled.
"""
def validate_instances(instances):
assert instances[str] == "foo"
assert instances[int] == 3
assert instances[float] == 3.0
providers = {
get_provider_for_cache({str}, "str", content="foo"): 1,
get_provider_for_cache({int, float}, "number", content=3): 2,
}
cache = tmp_path / "cache3.sqlite3"
if cache.exists():
print(f"Cache file {cache} already exists. Weird. Deleting")
cache.unlink()
settings = {"SCRAPY_POET_CACHE": cache,
"SCRAPY_POET_CACHE_ERRORS": cache_errors}
injector = get_injector_for_testing(providers, settings)
assert cache.exists()
def callback(response: DummyResponse, arg_str: str, arg_int: int, arg_float: float):
pass
response = get_response_for_testing(callback)
plan = injector.build_plan(response.request)
instances = yield from injector.build_instances_from_providers(
response.request, response, plan)
validate_instances(instances)
# Changing the request URL below would result in the following error:
# <twisted.python.failure.Failure builtins.ValueError: The URL is not from
# example.com>>
response.request = Request.replace(response.request, url="http://willfail.page")
with pytest.raises(ValueError):
plan = injector.build_plan(response.request)
instances = yield from injector.build_instances_from_providers(
response.request, response, plan)
# Different providers. They return a different result, but the cache data should prevail.
providers = {
get_provider_for_cache({str}, "str", content="bar", error=KeyError): 1,
get_provider_for_cache({int, float}, "number", content=4, error=KeyError): 2,
}
injector = get_injector_for_testing(providers, settings)
response = get_response_for_testing(callback)
plan = injector.build_plan(response.request)
instances = yield from injector.build_instances_from_providers(
response.request, response, plan)
validate_instances(instances)
# If caching errors is disabled, then KeyError should be raised.
Error = ValueError if cache_errors else KeyError
response.request = Request.replace(response.request, url="http://willfail.page")
with pytest.raises(Error):
plan = injector.build_plan(response.request)
instances = yield from injector.build_instances_from_providers(
response.request, response, plan)