def test_lifetime(self):
mock_r = MockRenderer({})
mock_r.render = MagicMock(side_effect=['result'])
mock_r.shutdown = MagicMock()
r = Lifetime(mock_r)
self.assertEqual(mock_r.render.call_count, 0)
self.assertEqual(mock_r.shutdown.call_count, 0)
self.assertEqual(r.render(u'http://test'), 'result')
self.assertEqual(mock_r.render.call_count, 1)
self.assertEqual(mock_r.shutdown.call_count, 0)
r.shutdown()
self.assertEqual(mock_r.render.call_count, 1)
self.assertEqual(mock_r.shutdown.call_count, 1)
def test_lifetime_max(self):
mock_r = MockRenderer({
'idle_shutdown_sec': 60,
'max_lifetime_sec': 0.5
})
mock_r.render = MagicMock()
mock_r.shutdown = MagicMock()
r = Lifetime(mock_r)
self.assertEqual(mock_r.render.call_count, 0)
self.assertEqual(mock_r.shutdown.call_count, 0)
eventlet.sleep(0.25)
r.render(u'http://test')
self.assertEqual(mock_r.render.call_count, 1)
self.assertEqual(mock_r.shutdown.call_count, 0)
eventlet.sleep(0.5)
r.render(u'http://test')
self.assertEqual(mock_r.render.call_count, 2)
self.assertEqual(mock_r.shutdown.call_count, 1)
eventlet.sleep(0.5)
r.render(u'http://test')
self.assertEqual(mock_r.render.call_count, 3)
self.assertEqual(mock_r.shutdown.call_count, 2)
r.shutdown()
self.assertEqual(mock_r.render.call_count, 3)
self.assertEqual(mock_r.shutdown.call_count, 3)
config = {
'executable': '/usr/local/bin/phantomjs',
'args':
PHANTOMJS['args'] + ['--disk-cache=false', '--load-images=true'],
'env': {
'TZ': 'America/Los_Angeles'
},
'timeouts': {
'process_startup': 100
},
'idle_shutdown_sec': 10,
'max_lifetime_sec': 5
}
r = PhantomJSRenderer(config)
r = Lifetime(r)
with open('/tmp/crawl.html', 'r') as content_file:
html = content_file.read()
html = html.decode('utf-8', errors='replace')
urls = [('http://www.some-domain.com', html)]
try:
for url in urls:
html = None
if isinstance(url, tuple):
html = url[1]
url = url[0]
print "Requesting {}".format(url)