def trial_split_consistency(data1, data2, metric, split_field,
image_property, response_property, kwargs=None, bstrapiter=900):
metric_func, kwargs = u.get_rm_metric(metric, kwargs)
split_field_vals = None
if split_field is not None:
split_field_vals = np.unique(data1[split_field])
CMS1 = get_rms(data1, split_field, image_property, response_property, split_field_vals)
CMS2 = get_rms(data2, split_field, image_property, response_property, split_field_vals)
m1 = []
m2 = []
for CM1, CM2 in zip(CMS1, CMS2):
m1.extend(metric_func(CM1, **kwargs))
m2.extend(metric_func(CM2, **kwargs))
R, _ = scipy.stats.spearmanr(m1, m2)
consistencies = []
rng = np.random.RandomState(0)
ICs1 = []
ICs2 = []
for rep in range(bstrapiter):
if rep % 100 == 0:
print rep / float(bstrapiter)
# Split first data
inds1 = rng.permutation(range(data1.shape[0]))
inds11, inds12 = inds1[:(inds1.shape[0] / 2)], inds1[(inds1.shape[0] / 2):]
RMs11 = get_rms(data1[inds11], split_field, image_property, response_property)
RMs12 = get_rms(data1[inds12], split_field, image_property, response_property)
#Split second data
inds2 = rng.permutation(range(data1.shape[0]))
inds21, inds22 = inds2[:(inds2.shape[0] / 2)], inds2[(inds2.shape[0] / 2):]
RMs21 = get_rms(data1[inds21], split_field, image_property, response_property)
RMs22 = get_rms(data1[inds22], split_field, image_property, response_property)
#Calculate metrics
m11, m12, m21, m22 = tuple([apply_metric(RMs, metric_func, kwargs) for RMs in [RMs11, RMs12, RMs21, RMs22]])
#noise level
IC1, _ = scipy.stats.spearmanr(m11, m12)
IC2, _ = scipy.stats.spearmanr(m21, m22)
noise = np.sqrt(IC1 * IC2)
#Consistencies
R1, _ = scipy.stats.spearmanr(m11, m21)
R2, _ = scipy.stats.spearmanr(m11, m22)
R3, _ = scipy.stats.spearmanr(m12, m21)
R4, _ = scipy.stats.spearmanr(m12, m22)
ICs1.append(IC1)
ICs2.append(IC2)
consistencies.extend([R1 / noise, R2 / noise, R3 / noise, R4 / noise])
print np.mean(consistencies), np.std(consistencies), np.mean(ICs1), np.std(ICs1), np.mean(ICs2), np.std(ICs2)
return np.mean(consistencies), np.std(consistencies), np.mean(ICs1), np.std(ICs1), np.mean(ICs2), np.std(ICs2)
def rms_spearman_consistency(RMs1, RMs2, metric, metrickwargs):
"""
Calculates the spearman consistency between two sets of metrics
:param RMs: List of matrices, each of which is [n_image_properties, n_response_properties, n_subjects]
:param metric: A response matrix metric which is registered in the method RM_metrics
:param metrickwargs: keyword arguments for the metric
"""
metric_func, kwargs = get_rm_metric(metric, metrickwargs)
m1 = []
m2 = []
for M1, M2 in zip(RMs1, RMs2):
m1.extend(metric_func(M1, **kwargs))
m2.extend(metric_func(M2, **kwargs))
return scipy.stats.spearmanr(np.ravel(m1), np.ravel(m2))[0]
def rms_spearman_consistency(RMs1, RMs2, metric, metrickwargs):
"""
Calculates the spearman consistency between two sets of metrics
:param RMs: List of matrices, each of which is [n_image_properties, n_response_properties, n_subjects]
:param metric: A response matrix metric which is registered in the method RM_metrics
:param metrickwargs: keyword arguments for the metric
"""
metric_func, kwargs = get_rm_metric(metric, metrickwargs)
m1 = []
m2 = []
for M1, M2 in zip(RMs1, RMs2):
m1.extend(metric_func(M1, **kwargs))
m2.extend(metric_func(M2, **kwargs))
return scipy.stats.spearmanr(np.ravel(m1), np.ravel(m2))[0]
def trial_split_half_consistency(trials,
metric,
kwargs,
split_field,
image_property,
response_property,
bstrapiter=900,
rng=None,
spearman_brown_correction=True):
"""
:param trials: Data in trial tabular array format as returned by dldata.confusion_matrices.get_data
:param metric: Oneof the metrics registered in dldata.metrics.utils get_rm_metrics()
:param kwargs: Kwargs to pass that metric
:param split_field: Generate one response matrix per unique value of this field in trials
:param image_property: What property of the image to use for response matrix
:param response_property: What property of the response to use for the response matrix
:param bstrapiter: NUmber of iterations to repeat bootstrap
:param rng: random number generator, as in np.random.RandomState
:param spearman_brown_correction: Whether to correct IC using spearman-brown prediction formula
:return: mean, standard deviation over bootstrap
"""
metric_func, kwargs = u.get_rm_metric(metric, kwargs)
if rng is None:
rng = np.random.RandomState(0)
ICs = []
for rep in range(bstrapiter):
if rep % 100 == 0:
print rep / float(bstrapiter)
inds = rng.permutation(range(trials.shape[0]))
inds1, inds2 = inds[:inds.shape[0] / 2], inds[inds.shape[0] / 2:]
CMS1 = get_rms(trials[inds1], split_field, image_property,
response_property)
CMS2 = get_rms(trials[inds2], split_field, image_property,
response_property)
m1 = []
m2 = []
for CM1, CM2 in zip(CMS1, CMS2):
m1.extend(metric_func(CM1, **kwargs))
m2.extend(metric_func(CM2, **kwargs))
IC, _ = scipy.stats.spearmanr(m1, m2)
if spearman_brown_correction:
IC = spearman_brown_correct(IC)
ICs.append(IC)
return np.mean(ICs), np.std(ICs)
def trial_split_half_consistency(trials, metric, kwargs, split_field,
image_property, response_property, bstrapiter=900, rng=None,
spearman_brown_correction=True):
"""
:param trials: Data in trial tabular array format as returned by dldata.confusion_matrices.get_data
:param metric: Oneof the metrics registered in dldata.metrics.utils get_rm_metrics()
:param kwargs: Kwargs to pass that metric
:param split_field: Generate one response matrix per unique value of this field in trials
:param image_property: What property of the image to use for response matrix
:param response_property: What property of the response to use for the response matrix
:param bstrapiter: NUmber of iterations to repeat bootstrap
:param rng: random number generator, as in np.random.RandomState
:param spearman_brown_correction: Whether to correct IC using spearman-brown prediction formula
:return: mean, standard deviation over bootstrap
"""
metric_func, kwargs = u.get_rm_metric(metric, kwargs)
if rng is None:
rng = np.random.RandomState(0)
ICs = []
for rep in range(bstrapiter):
if rep % 100 == 0:
print rep / float(bstrapiter)
inds = rng.permutation(range(trials.shape[0]))
inds1, inds2 = inds[:inds.shape[0] / 2], inds[inds.shape[0] / 2:]
CMS1 = get_rms(trials[inds1], split_field, image_property, response_property)
CMS2 = get_rms(trials[inds2], split_field, image_property, response_property)
m1 = []
m2 = []
for CM1, CM2 in zip(CMS1, CMS2):
m1.extend(metric_func(CM1, **kwargs))
m2.extend(metric_func(CM2, **kwargs))
IC, _ = scipy.stats.spearmanr(m1, m2)
if spearman_brown_correction:
IC = spearman_brown_correct(IC)
ICs.append(IC)
return np.mean(ICs), np.std(ICs)
def trial_split_consistency(data1,
data2,
metric,
split_field,
image_property,
response_property,
kwargs=None,
bstrapiter=900):
metric_func, kwargs = u.get_rm_metric(metric, kwargs)
split_field_vals = None
if split_field is not None:
split_field_vals = np.unique(data1[split_field])
CMS1 = get_rms(data1, split_field, image_property, response_property,
split_field_vals)
CMS2 = get_rms(data2, split_field, image_property, response_property,
split_field_vals)
m1 = []
m2 = []
for CM1, CM2 in zip(CMS1, CMS2):
m1.extend(metric_func(CM1, **kwargs))
m2.extend(metric_func(CM2, **kwargs))
R, _ = scipy.stats.spearmanr(m1, m2)
consistencies = []
rng = np.random.RandomState(0)
ICs1 = []
ICs2 = []
for rep in range(bstrapiter):
if rep % 100 == 0:
print rep / float(bstrapiter)
# Split first data
inds1 = rng.permutation(range(data1.shape[0]))
inds11, inds12 = inds1[:(inds1.shape[0] / 2)], inds1[(inds1.shape[0] /
2):]
RMs11 = get_rms(data1[inds11], split_field, image_property,
response_property)
RMs12 = get_rms(data1[inds12], split_field, image_property,
response_property)
#Split second data
inds2 = rng.permutation(range(data1.shape[0]))
inds21, inds22 = inds2[:(inds2.shape[0] / 2)], inds2[(inds2.shape[0] /
2):]
RMs21 = get_rms(data1[inds21], split_field, image_property,
response_property)
RMs22 = get_rms(data1[inds22], split_field, image_property,
response_property)
#Calculate metrics
m11, m12, m21, m22 = tuple([
apply_metric(RMs, metric_func, kwargs)
for RMs in [RMs11, RMs12, RMs21, RMs22]
])
#noise level
IC1, _ = scipy.stats.spearmanr(m11, m12)
IC2, _ = scipy.stats.spearmanr(m21, m22)
noise = np.sqrt(IC1 * IC2)
#Consistencies
R1, _ = scipy.stats.spearmanr(m11, m21)
R2, _ = scipy.stats.spearmanr(m11, m22)
R3, _ = scipy.stats.spearmanr(m12, m21)
R4, _ = scipy.stats.spearmanr(m12, m22)
ICs1.append(IC1)
ICs2.append(IC2)
consistencies.extend([R1 / noise, R2 / noise, R3 / noise, R4 / noise])
print np.mean(consistencies), np.std(consistencies), np.mean(ICs1), np.std(
ICs1), np.mean(ICs2), np.std(ICs2)
return np.mean(consistencies), np.std(consistencies), np.mean(
ICs1), np.std(ICs1), np.mean(ICs2), np.std(ICs2)
