def test_ok(self):
p=self.run_server([])
c=HTTPClient(timeout=2)
res = c.open_url('http://localhost:5000')
data = res.read()
self.assertTrue(len(data)>1000)
self.stop_server(p)
def run(plugin, base_dir, options):
if hasattr(plugin, 'init_plugin'):
plugin.init_plugin()
pool = None
try:
client = HTTPClient(options.proxy, options.no_proxy, plugin.REPEATS,
plugin.MEAN_WAIT, plugin.MAX_WAIT)
client2 = HTTPClient(options.proxy, options.no_proxy, plugin.REPEATS2,
plugin.MEAN_WAIT2, plugin.MAX_WAIT2)
pool = ThreadPool(os.path.join(base_dir, 'pool_items'),
functools.partial(plugin.save_file,
client2), plugin.DOWN_THREADS,
options.resume, plugin.MAX_QUEUE_SIZE)
timer = None
if options.stop_after:
timer = threading.Timer(int(options.stop_after) * 60,
interrupt,
args=[pool, client, client2])
timer.daemon = True
timer.start()
def stop_now(*args):
interrupt(pool, client, client2)
signal.signal(signal.SIGUSR1, stop_now)
spider = plugin.MySpider(
client, plugin.START_URL,
os.path.join(base_dir, 'last_page.txt')
if options.resume else None)
try:
for link, metadata in spider:
logging.debug('Got id %s' % metadata['id'])
pool.add_task(link, metadata, base_dir)
except Interrupted:
logging.info('Main loop interrupted - leaving')
try:
pool.wait_completion(not options.daemon)
logging.debug("Pool should finish all tasks - %d" %
pool.tasks.unfinished_tasks)
except Interrupted:
logging.info('Waiting for threads to complete')
pool.join(not options.daemon)
except Interrupted:
logging.info("Interruped at early stage")
except Exception:
logging.exception("Downloader run exits with error")
return True
except (SystemExit, KeyboardInterrupt) as e:
logging.info("terminating by %s", e)
if pool:
pool.close()
raise e
finally:
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
if hasattr(plugin, 'close_plugin'):
plugin.close_plugin()
def test_head(self):
p=self.run_server(['--header-delay', '5'])
c=HTTPClient(timeout=2)
try:
res = c.open_url('http://localhost:5000')
self.fail('Should raise error')
except HTTPClient.Error:
pass
self.stop_server(p)
def test_read(self):
p=self.run_server(['--send-delay', '5'])
c=HTTPClient(timeout=2)
res = c.open_url('http://localhost:5000')
try:
data = res.read()
self.fail('Should raise error')
except socket.timeout:
pass
self.stop_server(p)