def test_required_fields_attribute_exception(self): spider = BasicParseTestSpider() spider.pipelines = [RequiredFields] pipeline = RequiredFields() response = fake_response("basic_parse.html") with self.assertRaises(AttributeError): results = [pipeline.process_item(item, spider) for item in spider.parse(response)]
def test_basic_parse(self): spider = BasicParseTestSpider() response = fake_response("basic_parse.html") results = [item for item in spider.parse(response)] self.assertEqual(len(results), 3) for result, expected in zip(results, self.expected_items): for key in result.keys(): self.assertEqual(result[key], expected[key])
def test_basic_parse(self): spider = BasicParseFirstElementTestSpider() response = fake_response("basic_parse.html") results = [item for item in spider.parse(response)] self.assertEqual(len(results), 3) for result, expected in zip(results, self.expected_items): for key in result.keys(): self.assertEqual(result[key], expected[key])
def test_min_target_fail(self): spider = BasicParseTestSpider() spider.pipelines = [MinTargets] spider.Meta.min_targets = 4 pipeline = MinTargets() response = fake_response("basic_some_missing.html") with self.assertRaises(DropItem): results = [pipeline.process_item(item, spider) for item in spider.parse(response)]
def test_json_spider(self): expected = [{"greeting": "hello", "word": "neck!!"}, {"greeting": "hello", "word": "neck!!"}] response = fake_response("test.json") spider = JsonTestSpider() results = [item for item in spider.parse(response)] self.assertEqual(len(results), 2) for result, expected in zip(results, expected): for key in result.keys(): self.assertEqual(result[key], expected[key])
def test_required_fields_fail(self): spider = BasicParseTestSpider() spider.Meta.required_fields = ["disclaimer", "discount", "image_url", "offer_url", "title"] spider.pipelines = [RequiredFields] pipeline = RequiredFields() response = fake_response("basic_some_missing.html") with self.assertRaises(DropItem): for item in spider.parse(response): pipeline.process_item(item, spider)
def test_min_target_success(self): spider = BasicParseTestSpider() spider.pipelines = [MinTargets] spider.Meta.min_targets = 4 pipeline = MinTargets() response = fake_response("basic_parse.html") try: results = [pipeline.process_item(item, spider) for item in spider.parse(response)] except DropItem: self.fail("min_target dropeed item when it shouldn't.")
def test_required_fields_success(self): spider = BasicParseTestSpider() spider.Meta.required_fields = ["disclaimer", "discount", "image_url", "offer_url", "title"] spider.pipelines = [RequiredFields] response = fake_response("basic_parse.html") pipeline = RequiredFields() try: results = [pipeline.process_item(item, spider) for item in spider.parse(response)] except DropItem: self.fail("Valid required fields dropped item.") self.assertEqual(len(results), 3)