def test_item_validated_and_returned(self, item, verbose):
"""Verify that when the ValidationPipeline receives a correctly-constructed item, it returns the very same item after validating it.
"""
spider = UsesthisSpider('usesthis')
pipeline = ValidationPipeline()
pipeline._verbose = verbose
same_item = pipeline.process_item(item, spider)
self.assertEquals(same_item, item)
def test_drop_person_missing_field(self, item, verbose):
"""Verify that the ValidationPipeline drops the PersonItem instance if it's missing one or more fields.
"""
spider = UsesthisSpider('usesthis')
pipeline = ValidationPipeline()
pipeline._verbose = verbose
with self.assertRaises(DropItem):
returned_item = pipeline.process_item(item, spider)
self.assertIsNone(returned_item)
def test_drop_tool_blank_reqd_field(self, item, verbose):
"""Verify that the ValidationPipeline drops a ToolItem instance if it has no tool_name or no tool_url.
"""
spider = UsesthisSpider('usesthis')
pipeline = ValidationPipeline()
pipeline._verbose = verbose
clean_item = pipeline.process_item(item, spider)
for tool_item in clean_item['tools']:
for val in tool_item.values():
self.assertTrue(val)
def test_drop_tools_missing_field(self, item, verbose):
"""Verify that the ValidationPipeline deletes tool_items if they're missing one or more fields.
"""
spider = UsesthisSpider('usesthis')
pipeline = ValidationPipeline()
pipeline._verbose = verbose
clean_item = pipeline.process_item(item, spider)
for tool_item in clean_item['tools']:
missing_fields = set(ToolItem.fields) - set(tool_item)
self.assertFalse(missing_fields)
def test_drop_tool_malformed_url(self, item, verbose):
"""Verify that the ValidationPipeline drops a ToolItem instance if it has a malformed tool_url.
"""
spider = UsesthisSpider('usesthis')
pipeline = ValidationPipeline()
pipeline._verbose = verbose
clean_item = pipeline.process_item(item, spider)
# The only remaining tools should be from the one 'correct' URL
for tool_item in clean_item['tools']:
tool_url = tool_item['tool_url']
self.assertEquals(tool_url, 'http://plumbertools.org/pipebuster5000')
def test_drop_person_blank_reqd_field(self, item, verbose):
"""Verify that the ValidationPipeline drops the PersonItem instance if one or more of the following fields are empty:
name, article_url, pub_date, img_src
"""
spider = UsesthisSpider('usesthis')
pipeline = ValidationPipeline()
pipeline._verbose = verbose
if '' in item['person'].values():
with self.assertRaises(DropItem):
returned_item = pipeline.process_item(item, spider)
self.assertIsNone(returned_item)
else:
self.assertEquals(pipeline.process_item(item, spider), item)
def test_drop_person_malformed_url_or_date(self, item, verbose):
"""Verify that the ValidationPipeline drops the PersonItem instance if it has a malformed url or date.
"""
spider = UsesthisSpider('usesthis')
pipeline = ValidationPipeline()
pipeline._verbose = verbose
try:
with self.assertRaises(DropItem):
returned_item = pipeline.process_item(item, spider)
self.assertIsNone(returned_item)
except AssertionError:
self.assertEquals(returned_item['person'], PersonItem(
name='Joe Schmoe',
article_url='https://usesthis.com/interviews/joe.schmoe/',
pub_date='2014-04-08',
title='Plumber',
img_src='https://usesthis.com/images/portraits/joe.schmoe.jpg',
bio='Hi, my name is Joe. People call me "Joe the Plumber".',
hardware='I use the PipeBuster5000. Best wrench on the market.',
software='I couldn\'t care less about software.',
dream='My dream setup would be a PipeBuster6000.',
))