def test_searches(self, in_filename, out_filename, poolsize=10): """ Takes a file where each line gives a search and a list of expected URLs. """ start_time = time.time() pool = eventlet.GreenPool(size=poolsize) inf = open(in_filename, 'r') r = csv.reader(inf, dialect='excel-tab') def doeeet(row): row = map( lambda x: x.strip(), map( lambda x: x.decode('utf-8'), row ) ) query = row[0] expected = row[1:] try: return (query, self.test_search(query, expected)) except: print >>sys.stderr, "Caught exception while testing", query.encode('utf-8') return (query, map(lambda x: None, expected)) print "Running searches..." print results = pool.imap(doeeet, r) with open(out_filename, 'w') as outf: w = csv.writer(outf, dialect='excel-tab') scores = [] for count, result in enumerate(results): score = calculate_score(result[1]) scores.append(score) row = [ result[0], score ] # the query and score row.extend(result[1]) # and the positions w.writerow(map(lambda x: unicode(x).encode('utf-8') if x is not None else "", row)) if len(result[0]) > 69: truncated_name = result[0][:69] + u"…" else: truncated_name = result[0] print u" %-70.70s %.2f" % (truncated_name, score) if count % 10 == 9: print print "Average score", float(sum(scores)) / len(scores) print inf.close() print print "Summary" print "-------" print run_time = time.time() - start_time print "Ran %i queries in %.2f seconds, average score %f." % (len(scores), run_time, float(sum(scores)) / len(scores)) print
def scoretest(): from searchtester.scoring import calculate_score import csv from optparse import OptionParser parser = OptionParser( usage="usage: %prog [options] resultsfile", description="Given a TSV file of searches and 0-indexed positions the best matches were found at, print an 'accuracy' score for each query. Prints a final summary including the average accuracy score.", ) (options, args) = parser.parse_args() if len(args) != 1: parser.error("Must provide a results file, from a previous searchtest run .") with open(args[0]) as f: print "Calculating scores..." print r = csv.reader(f, dialect='excel-tab') scores = [] for count, row in enumerate(r): score = calculate_score(row[2:]) scores.append(score) if len(row[0]) > 69: truncated_name = row[0][:69] + u"…" else: truncated_name = row[0] print u" %-70.70s %.2f" % (truncated_name, score) if count % 10 == 9: print print "Average score", float(sum(scores)) / len(scores) print print print "Summary" print "-------" print print "%i queries, average score %f." % (len(scores), float(sum(scores)) / len(scores)) print return 0
def test_middling(self): self.assertRoughlyEqual(0.9926, calculate_score([0, 1, 2, 3, 5])) self.assertRoughlyEqual(0.9800, calculate_score([0, 1, 2, 4, 3])) self.assertRoughlyEqual(0.8520, calculate_score([0, 2, 1, 3, 4])) self.assertRoughlyEqual(0.7976, calculate_score([0, 3, 1, 2, 4])) self.assertRoughlyEqual(0.7775, calculate_score([0, 4, 1, 2, 3])) self.assertRoughlyEqual(0.7775, calculate_score([0, 3, 1, 4, 2])) self.assertRoughlyEqual(0.5977, calculate_score([1, 0, 2, 3, 4])) self.assertRoughlyEqual(0.5032, calculate_score([1, 0, 4, 3, 2])) self.assertRoughlyEqual(0.4497, calculate_score([2, 0, 1, 3, 4])) self.assertRoughlyEqual(0.4432, calculate_score([1, 2, 0, 3])) self.assertRoughlyEqual(0.3881, calculate_score([1, 3, 0, 2])) self.assertRoughlyEqual(0.2935, calculate_score([2, 1, 0, 3])) self.assertRoughlyEqual(0.2273, calculate_score([2, 1, 3, 4, 0]))
def test_awful(self): self.assertEqual(0.0, calculate_score([None])) self.assertEqual(0.0, calculate_score([None, None])) self.assertEqual(0.0, calculate_score([None, None, None]))
def test_perfect(self): self.assertEqual(1.0, calculate_score([0])) self.assertEqual(1.0, calculate_score([0, 1])) self.assertEqual(1.0, calculate_score([0, 1, 2])) self.assertEqual(1.0, calculate_score([0, 1, 2, 3])) self.assertEqual(1.0, calculate_score([0, 1, 2, 3, 4]))