X = []
Y = []
for r in u.rated:
if r.has_response():
if r.pair_id in ratings:
X.append(ratings[r.pair_id])
Y.append(r.response)
d = abs(ratings[r.pair_id] - r.response)
deltas['all'].append(d)
deltas[r.condition].append(d)
deltas[(r.condition, (r.field == 'general'))].append(d)
else:
ratings[r.pair_id] = r.response
if len(X) == len(Y) == 5 and len(set(X)) > 1 and len(set(Y)) > 1:
rho = correlation.spearman_rho_tr(X, Y)
spearmans['all'].append(rho)
if u.mturk and not u.scholar:
spearmans['mturk'].append(rho)
if u.scholar and not u.mturk:
spearmans['scholar'].append(rho)
write('intra-rater spearman correlations:')
for c in spearmans:
write('\t%s: %.3f n=%d, dev=%.3f' % (
c,
utils.mean(spearmans[c]),
len(spearmans[c]),
utils.dev(spearmans[c])
))
n = 10
means = [[] for i in range(n)]
for (pair_id, ratings) in pair_ratings[ktype][condition].items():
total = 0
for i in range(n):
pm = pair_min_counts[pair_id]
means[i].append(utils.mean(utils.sample_with_replacement(ratings, pm)))
total += utils.mean(random.sample(ratings, pm))
pair_means[ktype][condition][pair_id] = 1.0 * total / n
if means:
spearmans = []
for X in means:
for Y in means:
if len(set(X)) >= 1 and len(set(Y)) >= 1:
spearmans.append(correlation.spearman_rho_tr(X, Y))
write('\t%s, %s: bootstrapped spearman: %.3f (dev=%.3f)' %
(condition, condition, utils.mean(spearmans), utils.dev(spearmans)))
for ktype in pair_means.keys():
write('question type: %s' % ktype)
for (condition1, consensus1) in pair_means[ktype].items():
for (condition2, consensus2) in pair_means[ktype].items():
if not consensus1 or not consensus2 or condition1 == condition2 or condition1 == 'all' or condition2 == 'all':
continue
write('\tcondition %s, %s' % (condition1, condition2))
common_ids = list(set(consensus1.keys()).intersection(consensus2.keys()))
X = [consensus1[pid] for pid in common_ids]
Y = [consensus2[pid] for pid in common_ids]
write('\tcondition %s, %s' % (condition1, condition2))
pearsons = []
spearmans = []
samples_per_user = 10
for u1 in users1:
for u2 in users2:
if id(u1) == id(u2):
continue
common_pair_ids = list(
set(u1.keys()).intersection(u2.keys()))
if len(common_pair_ids) >= 5:
up = [] # user pearson samples
us = [] # user spearman samples
for i in xrange(samples_per_user):
sample_pair_ids = random.sample(common_pair_ids, 5)
X = [u1[pid] for pid in sample_pair_ids]
Y = [u2[pid] for pid in sample_pair_ids]
if len(set(X)) != 1 and len(set(Y)) != 1:
p = correlation.pearson_rho(X, Y)
s = correlation.spearman_rho_tr(X, Y)
up.append(p)
us.append(s)
if up and us:
pearsons.append(utils.mean(up))
spearmans.append(utils.mean(us))
write('\t\td unique pairs of users: %d' % len(spearmans))
write('\t\tspearman: mean=%.3f, dev=%s' %
(utils.mean(spearmans), utils.dev(spearmans)))
write('\t\tpearson: mean=%.3f, dev=%s' %
(utils.mean(pearsons), utils.dev(pearsons)))
#!/usr/bin/python -O
import sys
import correlation
import utils
survey = utils.Survey()
wordsim = {}
for line in open('dat/general.txt'):
tokens = line.split('\t')
pair = tuple(sorted([t.strip() for t in tokens[:2]]))
wordsim[pair] = float(tokens[2])
for condition in ('all', 'mturk', 'scholar'):
ratings = survey.get_ratings_by_condition('general', condition)
X = []
Y = []
for (pair_id, pair_ratings) in ratings.items():
pair = tuple(sorted(survey.id_to_phrases(pair_id)))
if pair in wordsim and len(pair_ratings) > 10:
X.append(utils.mean([r.response for r in pair_ratings if r.has_response()]))
Y.append(wordsim[pair])
else:
print 'unknown pair:', pair
print condition, correlation.pearson_rho(X, Y), correlation.spearman_rho_tr(X, Y)
means = [[] for i in range(n)]
for (pair_id, ratings) in pair_ratings[ktype][condition].items():
total = 0
for i in range(n):
pm = pair_min_counts[pair_id]
means[i].append(
utils.mean(utils.sample_with_replacement(ratings, pm)))
total += utils.mean(random.sample(ratings, pm))
pair_means[ktype][condition][pair_id] = 1.0 * total / n
if means:
spearmans = []
for X in means:
for Y in means:
if len(set(X)) >= 1 and len(set(Y)) >= 1:
spearmans.append(correlation.spearman_rho_tr(X, Y))
write('\t%s, %s: bootstrapped spearman: %.3f (dev=%.3f)' %
(condition, condition, utils.mean(spearmans),
utils.dev(spearmans)))
for ktype in pair_means.keys():
write('question type: %s' % ktype)
for (condition1, consensus1) in pair_means[ktype].items():
for (condition2, consensus2) in pair_means[ktype].items():
if not consensus1 or not consensus2 or condition1 == condition2 or condition1 == 'all' or condition2 == 'all':
continue
write('\tcondition %s, %s' % (condition1, condition2))
common_ids = list(
set(consensus1.keys()).intersection(consensus2.keys()))
X = [consensus1[pid] for pid in common_ids]
