def trial_split_consistency(data1, data2, metric, split_field, image_property, response_property, kwargs=None, bstrapiter=900): metric_func, kwargs = u.get_rm_metric(metric, kwargs) split_field_vals = None if split_field is not None: split_field_vals = np.unique(data1[split_field]) CMS1 = get_rms(data1, split_field, image_property, response_property, split_field_vals) CMS2 = get_rms(data2, split_field, image_property, response_property, split_field_vals) m1 = [] m2 = [] for CM1, CM2 in zip(CMS1, CMS2): m1.extend(metric_func(CM1, **kwargs)) m2.extend(metric_func(CM2, **kwargs)) R, _ = scipy.stats.spearmanr(m1, m2) consistencies = [] rng = np.random.RandomState(0) ICs1 = [] ICs2 = [] for rep in range(bstrapiter): if rep % 100 == 0: print rep / float(bstrapiter) # Split first data inds1 = rng.permutation(range(data1.shape[0])) inds11, inds12 = inds1[:(inds1.shape[0] / 2)], inds1[(inds1.shape[0] / 2):] RMs11 = get_rms(data1[inds11], split_field, image_property, response_property) RMs12 = get_rms(data1[inds12], split_field, image_property, response_property) #Split second data inds2 = rng.permutation(range(data1.shape[0])) inds21, inds22 = inds2[:(inds2.shape[0] / 2)], inds2[(inds2.shape[0] / 2):] RMs21 = get_rms(data1[inds21], split_field, image_property, response_property) RMs22 = get_rms(data1[inds22], split_field, image_property, response_property) #Calculate metrics m11, m12, m21, m22 = tuple([apply_metric(RMs, metric_func, kwargs) for RMs in [RMs11, RMs12, RMs21, RMs22]]) #noise level IC1, _ = scipy.stats.spearmanr(m11, m12) IC2, _ = scipy.stats.spearmanr(m21, m22) noise = np.sqrt(IC1 * IC2) #Consistencies R1, _ = scipy.stats.spearmanr(m11, m21) R2, _ = scipy.stats.spearmanr(m11, m22) R3, _ = scipy.stats.spearmanr(m12, m21) R4, _ = scipy.stats.spearmanr(m12, m22) ICs1.append(IC1) ICs2.append(IC2) consistencies.extend([R1 / noise, R2 / noise, R3 / noise, R4 / noise]) print np.mean(consistencies), np.std(consistencies), np.mean(ICs1), np.std(ICs1), np.mean(ICs2), np.std(ICs2) return np.mean(consistencies), np.std(consistencies), np.mean(ICs1), np.std(ICs1), np.mean(ICs2), np.std(ICs2)
def rms_spearman_consistency(RMs1, RMs2, metric, metrickwargs): """ Calculates the spearman consistency between two sets of metrics :param RMs: List of matrices, each of which is [n_image_properties, n_response_properties, n_subjects] :param metric: A response matrix metric which is registered in the method RM_metrics :param metrickwargs: keyword arguments for the metric """ metric_func, kwargs = get_rm_metric(metric, metrickwargs) m1 = [] m2 = [] for M1, M2 in zip(RMs1, RMs2): m1.extend(metric_func(M1, **kwargs)) m2.extend(metric_func(M2, **kwargs)) return scipy.stats.spearmanr(np.ravel(m1), np.ravel(m2))[0]
def rms_spearman_consistency(RMs1, RMs2, metric, metrickwargs): """ Calculates the spearman consistency between two sets of metrics :param RMs: List of matrices, each of which is [n_image_properties, n_response_properties, n_subjects] :param metric: A response matrix metric which is registered in the method RM_metrics :param metrickwargs: keyword arguments for the metric """ metric_func, kwargs = get_rm_metric(metric, metrickwargs) m1 = [] m2 = [] for M1, M2 in zip(RMs1, RMs2): m1.extend(metric_func(M1, **kwargs)) m2.extend(metric_func(M2, **kwargs)) return scipy.stats.spearmanr(np.ravel(m1), np.ravel(m2))[0]
def trial_split_half_consistency(trials, metric, kwargs, split_field, image_property, response_property, bstrapiter=900, rng=None, spearman_brown_correction=True): """ :param trials: Data in trial tabular array format as returned by dldata.confusion_matrices.get_data :param metric: Oneof the metrics registered in dldata.metrics.utils get_rm_metrics() :param kwargs: Kwargs to pass that metric :param split_field: Generate one response matrix per unique value of this field in trials :param image_property: What property of the image to use for response matrix :param response_property: What property of the response to use for the response matrix :param bstrapiter: NUmber of iterations to repeat bootstrap :param rng: random number generator, as in np.random.RandomState :param spearman_brown_correction: Whether to correct IC using spearman-brown prediction formula :return: mean, standard deviation over bootstrap """ metric_func, kwargs = u.get_rm_metric(metric, kwargs) if rng is None: rng = np.random.RandomState(0) ICs = [] for rep in range(bstrapiter): if rep % 100 == 0: print rep / float(bstrapiter) inds = rng.permutation(range(trials.shape[0])) inds1, inds2 = inds[:inds.shape[0] / 2], inds[inds.shape[0] / 2:] CMS1 = get_rms(trials[inds1], split_field, image_property, response_property) CMS2 = get_rms(trials[inds2], split_field, image_property, response_property) m1 = [] m2 = [] for CM1, CM2 in zip(CMS1, CMS2): m1.extend(metric_func(CM1, **kwargs)) m2.extend(metric_func(CM2, **kwargs)) IC, _ = scipy.stats.spearmanr(m1, m2) if spearman_brown_correction: IC = spearman_brown_correct(IC) ICs.append(IC) return np.mean(ICs), np.std(ICs)
def trial_split_half_consistency(trials, metric, kwargs, split_field, image_property, response_property, bstrapiter=900, rng=None, spearman_brown_correction=True): """ :param trials: Data in trial tabular array format as returned by dldata.confusion_matrices.get_data :param metric: Oneof the metrics registered in dldata.metrics.utils get_rm_metrics() :param kwargs: Kwargs to pass that metric :param split_field: Generate one response matrix per unique value of this field in trials :param image_property: What property of the image to use for response matrix :param response_property: What property of the response to use for the response matrix :param bstrapiter: NUmber of iterations to repeat bootstrap :param rng: random number generator, as in np.random.RandomState :param spearman_brown_correction: Whether to correct IC using spearman-brown prediction formula :return: mean, standard deviation over bootstrap """ metric_func, kwargs = u.get_rm_metric(metric, kwargs) if rng is None: rng = np.random.RandomState(0) ICs = [] for rep in range(bstrapiter): if rep % 100 == 0: print rep / float(bstrapiter) inds = rng.permutation(range(trials.shape[0])) inds1, inds2 = inds[:inds.shape[0] / 2], inds[inds.shape[0] / 2:] CMS1 = get_rms(trials[inds1], split_field, image_property, response_property) CMS2 = get_rms(trials[inds2], split_field, image_property, response_property) m1 = [] m2 = [] for CM1, CM2 in zip(CMS1, CMS2): m1.extend(metric_func(CM1, **kwargs)) m2.extend(metric_func(CM2, **kwargs)) IC, _ = scipy.stats.spearmanr(m1, m2) if spearman_brown_correction: IC = spearman_brown_correct(IC) ICs.append(IC) return np.mean(ICs), np.std(ICs)
def trial_split_consistency(data1, data2, metric, split_field, image_property, response_property, kwargs=None, bstrapiter=900): metric_func, kwargs = u.get_rm_metric(metric, kwargs) split_field_vals = None if split_field is not None: split_field_vals = np.unique(data1[split_field]) CMS1 = get_rms(data1, split_field, image_property, response_property, split_field_vals) CMS2 = get_rms(data2, split_field, image_property, response_property, split_field_vals) m1 = [] m2 = [] for CM1, CM2 in zip(CMS1, CMS2): m1.extend(metric_func(CM1, **kwargs)) m2.extend(metric_func(CM2, **kwargs)) R, _ = scipy.stats.spearmanr(m1, m2) consistencies = [] rng = np.random.RandomState(0) ICs1 = [] ICs2 = [] for rep in range(bstrapiter): if rep % 100 == 0: print rep / float(bstrapiter) # Split first data inds1 = rng.permutation(range(data1.shape[0])) inds11, inds12 = inds1[:(inds1.shape[0] / 2)], inds1[(inds1.shape[0] / 2):] RMs11 = get_rms(data1[inds11], split_field, image_property, response_property) RMs12 = get_rms(data1[inds12], split_field, image_property, response_property) #Split second data inds2 = rng.permutation(range(data1.shape[0])) inds21, inds22 = inds2[:(inds2.shape[0] / 2)], inds2[(inds2.shape[0] / 2):] RMs21 = get_rms(data1[inds21], split_field, image_property, response_property) RMs22 = get_rms(data1[inds22], split_field, image_property, response_property) #Calculate metrics m11, m12, m21, m22 = tuple([ apply_metric(RMs, metric_func, kwargs) for RMs in [RMs11, RMs12, RMs21, RMs22] ]) #noise level IC1, _ = scipy.stats.spearmanr(m11, m12) IC2, _ = scipy.stats.spearmanr(m21, m22) noise = np.sqrt(IC1 * IC2) #Consistencies R1, _ = scipy.stats.spearmanr(m11, m21) R2, _ = scipy.stats.spearmanr(m11, m22) R3, _ = scipy.stats.spearmanr(m12, m21) R4, _ = scipy.stats.spearmanr(m12, m22) ICs1.append(IC1) ICs2.append(IC2) consistencies.extend([R1 / noise, R2 / noise, R3 / noise, R4 / noise]) print np.mean(consistencies), np.std(consistencies), np.mean(ICs1), np.std( ICs1), np.mean(ICs2), np.std(ICs2) return np.mean(consistencies), np.std(consistencies), np.mean( ICs1), np.std(ICs1), np.mean(ICs2), np.std(ICs2)