sekitei_segments.define_segments(qlink_urls[:curl_for_segments],
unk_urls[:curl_for_segments], MAX_QUOTA)
qlink_urls = [(1, url) for url in qlink_urls[curl_for_segments:]]
unk_urls = [(0, url) for url in unk_urls[curl_for_segments:]]
urls_mix = []
urls_mix.extend(qlink_urls)
urls_mix.extend(unk_urls)
random.shuffle(urls_mix)
urls_fetched = 0
qlinks_fetched = 0
qlinks_count = len(qlink_urls)
for url in urls_mix:
if sekitei_segments.fetch_url(url[1]):
urls_fetched += 1
qlinks_fetched += url[0]
if urls_fetched >= MAX_QUOTA:
break
t2 = time.time()
result.append((name, (t2 - t1), float(urls_fetched), float(qlinks_fetched),
float(qlinks_count)))
#break
print "=== Test summary ==="
out = ''
avg_qlink_rate = 0
avg_fetched = 0
for r in result:
out = r[0] + " t: " + str(r[1]) + " fetched rate: " + str(
r[2] / MAX_QUOTA) + " qlink rate:" + str(r[3] / r[4])
unk_urls[:curl_for_segments],
MAX_QUOTA);
qlink_urls = [(1, url) for url in qlink_urls[curl_for_segments:]];
unk_urls = [(0, url) for url in unk_urls[curl_for_segments:] ];
urls_mix = []
urls_mix.extend(qlink_urls);
urls_mix.extend(unk_urls);
random.shuffle(urls_mix)
urls_fetched = 0;
qlinks_fetched = 0
qlinks_count = len(qlink_urls);
for url in urls_mix:
if sekitei_segments.fetch_url(url[1]) :
urls_fetched +=1;
qlinks_fetched += url[0];
if urls_fetched >= MAX_QUOTA:
break;
t2 = time.time()
result.append((name, (t2-t1), float(urls_fetched), float(qlinks_fetched), float(qlinks_count) ) );
print "=== Test summary ===";
out= ''
avg_qlink_rate = 0
avg_fetched = 0;
for r in result:
out = r[0] + " t: " + str(r[1]) + " fetched rate: " + str(r[2]/MAX_QUOTA) + " qlink rate:" + str(r[3] / r[4] )
if r[1] > MAX_TIME: