def fetch(thread_index, fetch_queue, icons):
while 1:
sub, html = fetch_queue.get()
if not html: break
try:
icon = favicon(html)
if icon: icons[sub] = icon
except:
pass
# set timeout
try:
socket.setdefaulttimeout(float(config.feed_timeout()))
except:
pass
# (optionally) spawn threads, fetch pages
threads = {}
if int(config.spider_threads()):
for i in range(int(config.spider_threads())):
threads[i] = Thread(target=fetch, args=(i, fetch_queue, icons))
fetch_queue.put((None, None))
threads[i].start()
for i in range(int(config.spider_threads())):
threads[i].join()
else:
fetch_queue.put((None, None))
fetch(0, fetch_queue, icons)
def test_timeout(self):
self.assertEqual(30, config.feed_timeout())
def test_timeout(self):
self.assertEqual(30, config.feed_timeout())
