if len(X) == len(Y) == 5 and len(set(X)) > 1 and len(set(Y)) > 1:
rho = correlation.spearman_rho_tr(X, Y)
spearmans['all'].append(rho)
if u.mturk and not u.scholar:
spearmans['mturk'].append(rho)
if u.scholar and not u.mturk:
spearmans['scholar'].append(rho)
write('intra-rater spearman correlations:')
for c in spearmans:
write('\t%s: %.3f n=%d, dev=%.3f' % (
c,
utils.mean(spearmans[c]),
len(spearmans[c]),
utils.dev(spearmans[c])
))
for spec in utils.SPECIFICITIES:
write('mean absolute errors for %s questions:' % spec)
for c in utils.CONDITIONS:
diffs = []
for (pair_id, ratings) in s.get_ratings_by_condition(spec, c).items():
for reratings in utils.group_by(ratings, lambda r: r.user).values():
if len(reratings) >= 2:
diffs.append(abs(reratings[0].response - reratings[1].response))
if diffs:
write('\tcondition %s: mae=%.3f, n=%d, dev=%.3f' %
( c, utils.mean(diffs), len(diffs), utils.dev(diffs)))
f.close()
write('\tcondition %s, %s' % (condition1, condition2))
pearsons = []
spearmans = []
samples_per_user = 10
for u1 in users1:
for u2 in users2:
if id(u1) == id(u2):
continue
common_pair_ids = list(
set(u1.keys()).intersection(u2.keys()))
if len(common_pair_ids) >= 5:
up = [] # user pearson samples
us = [] # user spearman samples
for i in xrange(samples_per_user):
sample_pair_ids = random.sample(common_pair_ids, 5)
X = [u1[pid] for pid in sample_pair_ids]
Y = [u2[pid] for pid in sample_pair_ids]
if len(set(X)) != 1 and len(set(Y)) != 1:
p = correlation.pearson_rho(X, Y)
s = correlation.spearman_rho_tr(X, Y)
up.append(p)
us.append(s)
if up and us:
pearsons.append(utils.mean(up))
spearmans.append(utils.mean(us))
write('\t\td unique pairs of users: %d' % len(spearmans))
write('\t\tspearman: mean=%.3f, dev=%s' %
(utils.mean(spearmans), utils.dev(spearmans)))
write('\t\tpearson: mean=%.3f, dev=%s' %
(utils.mean(pearsons), utils.dev(pearsons)))
for (pair_id, ratings) in pair_ratings[ktype][condition].items():
total = 0
for i in range(n):
pm = pair_min_counts[pair_id]
means[i].append(utils.mean(utils.sample_with_replacement(ratings, pm)))
total += utils.mean(random.sample(ratings, pm))
pair_means[ktype][condition][pair_id] = 1.0 * total / n
if means:
spearmans = []
for X in means:
for Y in means:
if len(set(X)) >= 1 and len(set(Y)) >= 1:
spearmans.append(correlation.spearman_rho_tr(X, Y))
write('\t%s, %s: bootstrapped spearman: %.3f (dev=%.3f)' %
(condition, condition, utils.mean(spearmans), utils.dev(spearmans)))
for ktype in pair_means.keys():
write('question type: %s' % ktype)
for (condition1, consensus1) in pair_means[ktype].items():
for (condition2, consensus2) in pair_means[ktype].items():
if not consensus1 or not consensus2 or condition1 == condition2 or condition1 == 'all' or condition2 == 'all':
continue
write('\tcondition %s, %s' % (condition1, condition2))
common_ids = list(set(consensus1.keys()).intersection(consensus2.keys()))
X = [consensus1[pid] for pid in common_ids]
Y = [consensus2[pid] for pid in common_ids]
diffs = [(abs(x - y), x, y, pid) for (pid, x, y) in zip(common_ids, X, Y)]
diffs.sort()
def write(message):
print message
f.write(message + '\n')
for ktype in utils.SPECIFICITIES:
for condition in utils.CONDITIONS:
pair_ratings = s.get_ratings_by_condition(ktype, condition)
values = []
for ratings in pair_ratings.values():
values.extend([r.response for r in ratings])
if not values:
continue
hist = [0] * 6
for r in values:
hist[r] += 1
write('condition %s, %s:'% (ktype, condition))
write('\tn: %d' % len(values))
write('\tmean: %.3f' % utils.mean(values))
write('\tdev: %.3f' % utils.dev(values))
tokens = [
'%d=%d' % (i, hist[i])
for i in range(1,6)
]
write('\tcounts: %s' % string.join(tokens))
tokens = [
'%d=%.3f' % (i, 1.0 * hist[i] / len(values))
for i in range(1,6)
]
write('\thist: %s' % string.join(tokens))
s = utils.Survey()
def write(message):
print message
f.write(message + '\n')
for ktype in utils.SPECIFICITIES:
for condition in utils.CONDITIONS:
pair_ratings = s.get_ratings_by_condition(ktype, condition)
values = []
for ratings in pair_ratings.values():
values.extend([r.response for r in ratings])
if not values:
continue
hist = [0] * 6
for r in values:
hist[r] += 1
write('condition %s, %s:' % (ktype, condition))
write('\tn: %d' % len(values))
write('\tmean: %.3f' % utils.mean(values))
write('\tdev: %.3f' % utils.dev(values))
tokens = ['%d=%d' % (i, hist[i]) for i in range(1, 6)]
write('\tcounts: %s' % string.join(tokens))
tokens = [
'%d=%.3f' % (i, 1.0 * hist[i] / len(values)) for i in range(1, 6)
]
write('\thist: %s' % string.join(tokens))
for i in range(n):
pm = pair_min_counts[pair_id]
means[i].append(
utils.mean(utils.sample_with_replacement(ratings, pm)))
total += utils.mean(random.sample(ratings, pm))
pair_means[ktype][condition][pair_id] = 1.0 * total / n
if means:
spearmans = []
for X in means:
for Y in means:
if len(set(X)) >= 1 and len(set(Y)) >= 1:
spearmans.append(correlation.spearman_rho_tr(X, Y))
write('\t%s, %s: bootstrapped spearman: %.3f (dev=%.3f)' %
(condition, condition, utils.mean(spearmans),
utils.dev(spearmans)))
for ktype in pair_means.keys():
write('question type: %s' % ktype)
for (condition1, consensus1) in pair_means[ktype].items():
for (condition2, consensus2) in pair_means[ktype].items():
if not consensus1 or not consensus2 or condition1 == condition2 or condition1 == 'all' or condition2 == 'all':
continue
write('\tcondition %s, %s' % (condition1, condition2))
common_ids = list(
set(consensus1.keys()).intersection(consensus2.keys()))
X = [consensus1[pid] for pid in common_ids]
Y = [consensus2[pid] for pid in common_ids]
diffs = [(abs(x - y), x, y, pid)
deltas[(r.condition, (r.field == 'general'))].append(d)
else:
ratings[r.pair_id] = r.response
if len(X) == len(Y) == 5 and len(set(X)) > 1 and len(set(Y)) > 1:
rho = correlation.spearman_rho_tr(X, Y)
spearmans['all'].append(rho)
if u.mturk and not u.scholar:
spearmans['mturk'].append(rho)
if u.scholar and not u.mturk:
spearmans['scholar'].append(rho)
write('intra-rater spearman correlations:')
for c in spearmans:
write('\t%s: %.3f n=%d, dev=%.3f' % (c, utils.mean(
spearmans[c]), len(spearmans[c]), utils.dev(spearmans[c])))
for spec in utils.SPECIFICITIES:
write('mean absolute errors for %s questions:' % spec)
for c in utils.CONDITIONS:
diffs = []
for (pair_id, ratings) in s.get_ratings_by_condition(spec, c).items():
for reratings in utils.group_by(ratings,
lambda r: r.user).values():
if len(reratings) >= 2:
diffs.append(
abs(reratings[0].response - reratings[1].response))
if diffs:
write('\tcondition %s: mae=%.3f, n=%d, dev=%.3f' %
(c, utils.mean(diffs), len(diffs), utils.dev(diffs)))
f.close()
for (condition2, users2) in users[ktype].items():
if not users1 or not users2:
continue
write('\tcondition %s, %s' % (condition1, condition2))
pearsons = []
spearmans = []
samples_per_user = 10
for u1 in users1:
for u2 in users2:
if id(u1) == id(u2):
continue
common_pair_ids = list(set(u1.keys()).intersection(u2.keys()))
if len(common_pair_ids) >= 5:
up = [] # user pearson samples
us = [] # user spearman samples
for i in xrange(samples_per_user):
sample_pair_ids = random.sample(common_pair_ids, 5)
X = [u1[pid] for pid in sample_pair_ids]
Y = [u2[pid] for pid in sample_pair_ids]
if len(set(X)) != 1 and len(set(Y)) != 1:
p = correlation.pearson_rho(X, Y)
s = correlation.spearman_rho_tr(X, Y)
up.append(p)
us.append(s)
if up and us:
pearsons.append(utils.mean(up))
spearmans.append(utils.mean(us))
write('\t\td unique pairs of users: %d' % len(spearmans))
write('\t\tspearman: mean=%.3f, dev=%s' % (utils.mean(spearmans), utils.dev(spearmans)))
write('\t\tpearson: mean=%.3f, dev=%s' % (utils.mean(pearsons), utils.dev(pearsons)))
