def test_error(self):
if sys.version_info < (3, ):
self.skipTest('Unsupported Mock in Python 2.7')
m = Mock(side_effect=KeyError)
with patch('traceback.print_exc') as mock_print_exc:
with self.assertRaises(KeyError):
reraise_with_stack(m)()
m.assert_called_once()
mock_print_exc.assert_called_once()
def add_url(self, crawler_url, force=False):
"""Add url to queue"""
if not isinstance(crawler_url, CrawlerUrl):
crawler_url = CrawlerUrl(self,
crawler_url,
depth=self.depth,
timeout=self.timeout)
self.add_lock.acquire()
url = crawler_url.url
if not url.is_valid() or not url.only_domain or not self.in_domains(
url.only_domain):
self.add_lock.release()
return
if url.url in self.processing or url.url in self.processed:
self.add_lock.release()
return self.processing.get(url.url) or self.processed.get(url.url)
fn = reraise_with_stack(crawler_url.start)
if self.closing:
self.add_lock.release()
return
if force:
future = ThreadPoolExecutor(max_workers=1).submit(fn)
else:
future = self.submit(fn)
self.processing[url.url] = future
self.add_lock.release()
return future
def execute(*args, **kwargs):
try:
return reraise_with_stack(self.callback)(*args, **kwargs)
except Exception:
raise
finally:
self.threads_running -= 1
def submit(self, *args, **kwargs):
return super(Pool, self).submit(reraise_with_stack(self.callback),
*args, **kwargs)
def test_ok(self):
if sys.version_info < (3, ):
self.skipTest('Unsupported Mock in Python 2.7')
m = Mock()
reraise_with_stack(m)()
m.assert_called_once()