def check_calibration(world_prob):
intervals = np.arange(0, 1, .05)
res = np.array(world_prob)
calib = []
for start in intervals:
relevant = np.logical_and(res[:,1] > start, res[:,1] < start + .05)
actual = res[relevant, 0]
probs = res[relevant, 1]
print(sum(actual == False) / len(actual), jmutils.mean(probs))
calib.append([sum(actual == False) / len(actual), jmutils.mean(probs)])
return calib
def calc_easy_bts_binary(responses, meta_true):
"""Calculate easiest binary version of bts using own answers and predicted percentage of ppl endorsing true
Args:
responses: list where elt is 0 if subject said false and 1 if subject said true.
meta_true: list where elt is subject's prediction of fraction of ppl endorsing true.
Returns:
dict where each field (bts, surprise, accuracy) points to an np.array with an elt for each subject.
"""
assert len(responses) == len(meta_true)
do_assert = True
xbar_true = jmutils.mean(responses)
xbar_true = jmutils.bound(xbar_true, 0.01, 0.99)
xbar = [1 - xbar_true, xbar_true]
meta_true_trimmed = [jmutils.bound(mt, 0.01, 0.99) for mt in meta_true]
meta_false = [1 - et for et in meta_true_trimmed]
log_ybar = [jmutils.log_geo_mean(meta_false), jmutils.log_geo_mean(meta_true_trimmed)]
#if (xbar_true != jmutils.mean(responses)) or (len(responses) == 1):
# do_assert = False
easy_bts = calc_generic_bts(xbar, log_ybar, responses, zip(meta_false, meta_true_trimmed))
return easy_bts
