def from_crawler(cls, crawler):
parent = ItemValidationPipeline.from_crawler(crawler)
return cls(parent.validators,
crawler.stats,
parent.drop_items_with_errors,
parent.add_errors_to_items,
parent.errors_field)
def _function(self):
crawler = get_crawler(settings_dict=data_test.settings)
pipe = ItemValidationPipeline.from_crawler(crawler)
pipe.process_item(data_test.item, None)
kwargs = {"stats": "pipe.stats.stats.get_stats()"}
cases = data_test.cases
for case in cases if type(cases) in [list, tuple] else [cases]:
assert eval(case.format(**kwargs))
def test_return_pass_through_pipeline_if_spidermon_enabled_setting_is_not_provided():
settings = {
"EXTENSIONS": {"spidermon.contrib.scrapy.extensions.Spidermon": 100},
"SPIDERMON_VALIDATION_SCHEMAS": [{"dummy": "schema"}],
}
crawler = get_crawler(settings_dict=settings)
pipeline = ItemValidationPipeline.from_crawler(crawler)
assert isinstance(pipeline, PassThroughPipeline)
def test_spidermon_enabled_return_item_validation_pipeline():
settings = {
"SPIDERMON_ENABLED": True,
"EXTENSIONS": {"spidermon.contrib.scrapy.extensions.Spidermon": 100},
"SPIDERMON_VALIDATION_SCHEMAS": [{"dummy": "schema"}],
}
crawler = get_crawler(settings_dict=settings)
pipeline = ItemValidationPipeline.from_crawler(crawler)
assert isinstance(pipeline, ItemValidationPipeline)
def test_stats_amounts_in_pipeline():
item = TestItem({"title": "ScrapingHub"})
settings = {SETTING_CERBERUS: [cerberus_test_schema]}
crawler = get_crawler(settings_dict=settings)
pipe = ItemValidationPipeline.from_crawler(crawler)
pipe.process_item(item, None)
assert the_stats(pipe)["spidermon/validation/validators"] == 1
assert (the_stats(pipe)
["spidermon/validation/fields/errors/missing_required_field"] == 1)
def test_validator_from_url(mocker):
mocked_get_contents = mocker.patch(
"spidermon.contrib.validation.jsonschema.tools.get_contents",
return_value=test_schema_string,
)
settings = {
"SPIDERMON_ENABLED": True,
SETTING_SCHEMAS: {TestItem: "https://fixtures.com/testschema.json"},
}
test_item = TestItem()
crawler = get_crawler(settings_dict=settings)
pipe = ItemValidationPipeline.from_crawler(crawler)
pipe.process_item(test_item, None)
kwargs = {"stats": "pipe.stats.stats.get_stats()"}
stats = pipe.stats.stats.get_stats()
assert "spidermon/validation/items/errors" in stats
assert stats.get("spidermon/validation/validators/testitem/jsonschema", False)
def test_validation_from_url(mocker):
mocked_get_contents = mocker.patch(
"spidermon.contrib.validation.utils.get_contents",
return_value=test_cerberus_schema_string,
)
settings = {
SETTING_CERBERUS: {
TestItem: "https://fixtures.com/testschema.json"
}
}
test_item = TestItem()
crawler = get_crawler(settings_dict=settings)
pipe = ItemValidationPipeline.from_crawler(crawler)
pipe.process_item(test_item, None)
assert "spidermon/validation/items/errors" in the_stats(pipe)
assert the_stats(pipe).get(
"spidermon/validation/validators/testitem/cerberus", False)
def test_stats_in_pipeline(item, settings, cases):
crawler = get_crawler(settings_dict=settings)
pipe = ItemValidationPipeline.from_crawler(crawler)
pipe.process_item(item, None)
for case in cases:
assert case in the_stats(pipe)