def test_required_fields_attribute_exception(self):
spider = BasicParseTestSpider()
spider.pipelines = [RequiredFields]
pipeline = RequiredFields()
response = fake_response("basic_parse.html")
with self.assertRaises(AttributeError):
results = [pipeline.process_item(item, spider) for item in spider.parse(response)]
def test_basic_parse(self):
spider = BasicParseTestSpider()
response = fake_response("basic_parse.html")
results = [item for item in spider.parse(response)]
self.assertEqual(len(results), 3)
for result, expected in zip(results, self.expected_items):
for key in result.keys():
self.assertEqual(result[key], expected[key])
def test_basic_parse(self):
spider = BasicParseFirstElementTestSpider()
response = fake_response("basic_parse.html")
results = [item for item in spider.parse(response)]
self.assertEqual(len(results), 3)
for result, expected in zip(results, self.expected_items):
for key in result.keys():
self.assertEqual(result[key], expected[key])
def test_min_target_fail(self):
spider = BasicParseTestSpider()
spider.pipelines = [MinTargets]
spider.Meta.min_targets = 4
pipeline = MinTargets()
response = fake_response("basic_some_missing.html")
with self.assertRaises(DropItem):
results = [pipeline.process_item(item, spider) for item in spider.parse(response)]
def test_json_spider(self):
expected = [{"greeting": "hello", "word": "neck!!"}, {"greeting": "hello", "word": "neck!!"}]
response = fake_response("test.json")
spider = JsonTestSpider()
results = [item for item in spider.parse(response)]
self.assertEqual(len(results), 2)
for result, expected in zip(results, expected):
for key in result.keys():
self.assertEqual(result[key], expected[key])
def test_required_fields_fail(self):
spider = BasicParseTestSpider()
spider.Meta.required_fields = ["disclaimer", "discount", "image_url", "offer_url", "title"]
spider.pipelines = [RequiredFields]
pipeline = RequiredFields()
response = fake_response("basic_some_missing.html")
with self.assertRaises(DropItem):
for item in spider.parse(response):
pipeline.process_item(item, spider)
def test_min_target_success(self):
spider = BasicParseTestSpider()
spider.pipelines = [MinTargets]
spider.Meta.min_targets = 4
pipeline = MinTargets()
response = fake_response("basic_parse.html")
try:
results = [pipeline.process_item(item, spider) for item in spider.parse(response)]
except DropItem:
self.fail("min_target dropeed item when it shouldn't.")
def test_required_fields_success(self):
spider = BasicParseTestSpider()
spider.Meta.required_fields = ["disclaimer", "discount", "image_url", "offer_url", "title"]
spider.pipelines = [RequiredFields]
response = fake_response("basic_parse.html")
pipeline = RequiredFields()
try:
results = [pipeline.process_item(item, spider) for item in spider.parse(response)]
except DropItem:
self.fail("Valid required fields dropped item.")
self.assertEqual(len(results), 3)
