def check_calibration(world_prob): intervals = np.arange(0, 1, .05) res = np.array(world_prob) calib = [] for start in intervals: relevant = np.logical_and(res[:,1] > start, res[:,1] < start + .05) actual = res[relevant, 0] probs = res[relevant, 1] print(sum(actual == False) / len(actual), jmutils.mean(probs)) calib.append([sum(actual == False) / len(actual), jmutils.mean(probs)]) return calib
def calc_easy_bts_binary(responses, meta_true): """Calculate easiest binary version of bts using own answers and predicted percentage of ppl endorsing true Args: responses: list where elt is 0 if subject said false and 1 if subject said true. meta_true: list where elt is subject's prediction of fraction of ppl endorsing true. Returns: dict where each field (bts, surprise, accuracy) points to an np.array with an elt for each subject. """ assert len(responses) == len(meta_true) do_assert = True xbar_true = jmutils.mean(responses) xbar_true = jmutils.bound(xbar_true, 0.01, 0.99) xbar = [1 - xbar_true, xbar_true] meta_true_trimmed = [jmutils.bound(mt, 0.01, 0.99) for mt in meta_true] meta_false = [1 - et for et in meta_true_trimmed] log_ybar = [jmutils.log_geo_mean(meta_false), jmutils.log_geo_mean(meta_true_trimmed)] #if (xbar_true != jmutils.mean(responses)) or (len(responses) == 1): # do_assert = False easy_bts = calc_generic_bts(xbar, log_ybar, responses, zip(meta_false, meta_true_trimmed)) return easy_bts