def do_browse_work(url):
# Scrape the child urls first without going through the proxy.
_,_,_,child_urls = no_proxy_browser.visit(url, timeout=args.timeout)
workq.extend(child_urls)
info("Added %d urls to workq."%(len(child_urls)))
# Now generate load for the proxy.
if args.proxy:
br = Browser()
target_url = "%s/?q="%(args.proxy) + url
br.visit(target_url, timeout=args.timeout)
def measure_latency(latencies_by_url):
info("Measuring response times....")
br = Browser()
if args.proxy:
info("Proxy address: " + args.proxy)
for url, latencies in latencies_by_url.items():
target_url = url
if args.proxy:
target_url = "%s/?q="%(args.proxy) + url
start_time = time.time()
try:
_,_,html,_ = br.visit(target_url, timeout=args.timeout)
html_log.write("-"*75 + "\n")
html_log.write(url + "\n")
html_log.write(html + "\n")
except browser.TimeoutException:
warn("Timed out while visiting %s"%(target_url))
elapsed = "timeout"
else:
elapsed = time.time() - start_time
latencies.append(elapsed)
info("Measurement complete.")