コード例 #1
0
 def test_item_validated_and_returned(self, item, verbose):
     """Verify that when the ValidationPipeline receives a correctly-constructed item, it returns the very same item after validating it.
     """
     spider = UsesthisSpider('usesthis')
     pipeline = ValidationPipeline()
     pipeline._verbose = verbose
     same_item = pipeline.process_item(item, spider)
     self.assertEquals(same_item, item)
コード例 #2
0
 def test_drop_person_missing_field(self, item, verbose):
     """Verify that the ValidationPipeline drops the PersonItem instance if it's missing one or more fields.
     """
     spider = UsesthisSpider('usesthis')
     pipeline = ValidationPipeline()
     pipeline._verbose = verbose
     with self.assertRaises(DropItem):
         returned_item = pipeline.process_item(item, spider)
         self.assertIsNone(returned_item)
コード例 #3
0
 def test_drop_tool_blank_reqd_field(self, item, verbose):
     """Verify that the ValidationPipeline drops a ToolItem instance if it has no tool_name or no tool_url.
     """
     spider = UsesthisSpider('usesthis')
     pipeline = ValidationPipeline()
     pipeline._verbose = verbose
     clean_item = pipeline.process_item(item, spider)
     for tool_item in clean_item['tools']:
         for val in tool_item.values():
             self.assertTrue(val)
コード例 #4
0
 def test_drop_tools_missing_field(self, item, verbose):
     """Verify that the ValidationPipeline deletes tool_items if they're missing one or more fields.
     """
     spider = UsesthisSpider('usesthis')
     pipeline = ValidationPipeline()
     pipeline._verbose = verbose
     clean_item = pipeline.process_item(item, spider)
     for tool_item in clean_item['tools']:
         missing_fields = set(ToolItem.fields) - set(tool_item)
         self.assertFalse(missing_fields)
コード例 #5
0
 def test_drop_tool_malformed_url(self, item, verbose):
     """Verify that the ValidationPipeline drops a ToolItem instance if it has a malformed tool_url.
     """
     spider = UsesthisSpider('usesthis')
     pipeline = ValidationPipeline()
     pipeline._verbose = verbose
     clean_item = pipeline.process_item(item, spider)
     # The only remaining tools should be from the one 'correct' URL
     for tool_item in clean_item['tools']:
         tool_url = tool_item['tool_url']
         self.assertEquals(tool_url, 'http://plumbertools.org/pipebuster5000')
コード例 #6
0
 def test_drop_person_blank_reqd_field(self, item, verbose):
     """Verify that the ValidationPipeline drops the PersonItem instance if one or more of the following fields are empty:
     name, article_url, pub_date, img_src
     """
     spider = UsesthisSpider('usesthis')
     pipeline = ValidationPipeline()
     pipeline._verbose = verbose
     if '' in item['person'].values():
         with self.assertRaises(DropItem):
             returned_item = pipeline.process_item(item, spider)
             self.assertIsNone(returned_item)
     else:
         self.assertEquals(pipeline.process_item(item, spider), item)
コード例 #7
0
 def test_drop_person_malformed_url_or_date(self, item, verbose):
     """Verify that the ValidationPipeline drops the PersonItem instance if it has a malformed url or date.
     """
     spider = UsesthisSpider('usesthis')
     pipeline = ValidationPipeline()
     pipeline._verbose = verbose
     try:
         with self.assertRaises(DropItem):
             returned_item = pipeline.process_item(item, spider)
             self.assertIsNone(returned_item)
     except AssertionError:
         self.assertEquals(returned_item['person'], PersonItem(
             name='Joe Schmoe',
             article_url='https://usesthis.com/interviews/joe.schmoe/',
             pub_date='2014-04-08',
             title='Plumber',
             img_src='https://usesthis.com/images/portraits/joe.schmoe.jpg',
             bio='Hi, my name is Joe. People call me "Joe the Plumber".',
             hardware='I use the PipeBuster5000. Best wrench on the market.',
             software='I couldn\'t care less about software.',
             dream='My dream setup would be a PipeBuster6000.',
         ))