Example #1
0
 def test_execute_work(self, mock_wait_until_work_is_done, mock_add_pages_to_queue):
     mock_start_pages = mock_factory.create_mock_pages(10)
     self.work_executor.execute_work(mock_start_pages)
     self.assertEquals(self.work_executor.item_processor.open_output_file_if_needed.call_count, 1)
     self.assertEquals(mock_add_pages_to_queue.call_count, 1)
     self.assertEquals(mock_wait_until_work_is_done.call_count, 1)
     self.assertEquals(self.work_executor.item_processor.close_output_file_if_needed.call_count, 1)
Example #2
0
    def test_visit_return_list_pages(self, mock_visit):
        mock_page = mock.create_autospec(Page).return_value
        mock_pages = mock_factory.create_mock_pages(10)
        mock_pages_list = mock.Mock()
        mock_visit.return_value = mock_pages
        self.page_scraper.object_converter.convert_to_list_if_single_object.return_value = mock_pages_list
        pages_list = self.page_scraper.extract_pages_list(mock_page)

        mock_visit.assert_called_once_with(mock_page)
        self.assertEquals(pages_list, mock_pages_list)
Example #3
0
 def test_visit_return_list_pages(self, mock_visit):
     mock_page = mock.create_autospec(Page).return_value
     mock_pages = mock_factory.create_mock_pages(10)
     mock_pages_list = mock.Mock()
     mock_visit.return_value = mock_pages
     self.page_scraper.object_converter.convert_to_list_if_single_object.return_value = mock_pages_list
     pages_list = self.page_scraper.extract_pages_list(mock_page)
     
     mock_visit.assert_called_once_with(mock_page)
     self.assertEquals(pages_list, mock_pages_list)
Example #4
0
 def test_execute_work(self, mock_wait_until_work_is_done,
                       mock_add_pages_to_queue):
     mock_start_pages = mock_factory.create_mock_pages(10)
     self.work_executor.execute_work(mock_start_pages)
     self.assertEquals(
         self.work_executor.item_processor.open_output_file_if_needed.
         call_count, 1)
     self.assertEquals(mock_add_pages_to_queue.call_count, 1)
     self.assertEquals(mock_wait_until_work_is_done.call_count, 1)
     self.assertEquals(
         self.work_executor.item_processor.close_output_file_if_needed.
         call_count, 1)
Example #5
0
 def test_extract_pages(self):
     mock_pages_list = mock_factory.create_mock_pages(10)
     self.page.scraper.extract_pages_list.return_value = mock_pages_list
     pages_list = self.page.extract_pages()
     self.assertEquals(pages_list, mock_pages_list)
Example #6
0
 def setUp(self):
     start_pages = mock_factory.create_mock_pages(10)
     config = mock.create_autospec(Config).return_value
     work_executor_factory = mock.create_autospec(WorkExecutorFactory).return_value
     self.crawler = XCrawler(start_pages, config, work_executor_factory)
Example #7
0
 def test_add_pages_to_queue(self):
     mock_start_pages = mock_factory.create_mock_pages(10)
     self.work_executor.add_pages_to_queue(mock_start_pages)
     self.assertEquals(self.work_executor.page_queue.put.call_count,
                       len(mock_start_pages))
Example #8
0
 def test_add_pages_to_queue(self):
     mock_start_pages = mock_factory.create_mock_pages(10)
     self.work_executor.add_pages_to_queue(mock_start_pages)
     self.assertEquals(self.work_executor.page_queue.put.call_count, len(mock_start_pages))
Example #9
0
 def test_extract_pages(self):
     mock_pages_list = mock_factory.create_mock_pages(10)
     self.page.scraper.extract_pages_list.return_value = mock_pages_list
     pages_list = self.page.extract_pages()
     self.assertEquals(pages_list, mock_pages_list)