def cross_predict(model1, model2):
"""
Given two model objects, fit to the measurements conducted in one, and then
predict the measurements in the other model's rotational coordinate frame
(b vectors). Calculate relative RMSE on that prediction, relative to the
noise in the measurement due to the rotation. Average across both
directions of this operation.
"""
out = ozu.nans(model1.shape[:-1])
sig1 = model1.signal[model1.mask]
sig2 = model2.signal[model2.mask]
# Cross predict, using the parameters from one model to predict the
# measurements in the other model's b vectors:
predict1 = model1.predict(model2.bvecs[:, model2.b_idx])[model1.mask]
predict2 = model2.predict(model1.bvecs[:, model1.b_idx])[model2.mask]
signal_rmse = ozu.rmse(sig1, sig2)
predict1_rmse = ozu.rmse(predict1, sig2)
predict2_rmse = ozu.rmse(predict2, sig1)
# Average in each element:
predict_rmse = (predict1_rmse + predict2_rmse) / 2.
rel_rmse = predict_rmse / signal_rmse
out[model1.mask] = rel_rmse
return out
def overfitting_index(model1, model2):
"""
How badly is the model overfitting? This can be assessed by comparing the
RMSE of the model compared to the fit data (or learning set), relative to
the RMSE of the model on another data set (or testing set)
"""
sig1 = model1.signal[model1.mask]
sig2 = model2.signal[model2.mask]
fit1 = model1.fit[model1.mask]
fit2 = model2.fit[model2.mask]
rmse_train1 = ozu.rmse(fit1, sig1)
rmse_train2 = ozu.rmse(fit2, sig2)
rmse_test1 = ozu.rmse(fit1, sig2)
rmse_test2 = ozu.rmse(fit2, sig1)
fit_rmse = (rmse_train1 + rmse_train2) / 2.
predict_rmse = (rmse_test1 + rmse_test2) /2.
# The measure is a contrast index of the error on the training data vs. the
# error on the testing data:
overfit = (fit_rmse - predict_rmse) / (fit_rmse + predict_rmse)
out = ozu.nans(model1.shape[:-1])
out[model1.mask] = overfit
return out
def overfitting_index(model1, model2):
"""
How badly is the model overfitting? This can be assessed by comparing the
RMSE of the model compared to the fit data (or learning set), relative to
the RMSE of the model on another data set (or testing set)
"""
sig1 = model1.signal[model1.mask]
sig2 = model2.signal[model2.mask]
fit1 = model1.fit[model1.mask]
fit2 = model2.fit[model2.mask]
rmse_train1 = ozu.rmse(fit1, sig1)
rmse_train2 = ozu.rmse(fit2, sig2)
rmse_test1 = ozu.rmse(fit1, sig2)
rmse_test2 = ozu.rmse(fit2, sig1)
fit_rmse = (rmse_train1 + rmse_train2) / 2.
predict_rmse = (rmse_test1 + rmse_test2) / 2.
# The measure is a contrast index of the error on the training data vs. the
# error on the testing data:
overfit = (fit_rmse - predict_rmse) / (fit_rmse + predict_rmse)
out = ozu.nans(model1.shape[:-1])
out[model1.mask] = overfit
return out
def cross_predict(model1, model2):
"""
Given two model objects, fit to the measurements conducted in one, and then
predict the measurements in the other model's rotational coordinate frame
(b vectors). Calculate relative RMSE on that prediction, relative to the
noise in the measurement due to the rotation. Average across both
directions of this operation.
"""
out = ozu.nans(model1.shape[:-1])
sig1 = model1.signal[model1.mask]
sig2 = model2.signal[model2.mask]
# Cross predict, using the parameters from one model to predict the
# measurements in the other model's b vectors:
predict1 = model1.predict(model2.bvecs[:, model2.b_idx])[model1.mask]
predict2 = model2.predict(model1.bvecs[:, model1.b_idx])[model2.mask]
signal_rmse = ozu.rmse(sig1, sig2)
predict1_rmse = ozu.rmse(predict1, sig2)
predict2_rmse = ozu.rmse(predict2, sig1)
# Average in each element:
predict_rmse = (predict1_rmse + predict2_rmse) / 2.
rel_rmse = predict_rmse/signal_rmse
out[model1.mask] = rel_rmse
return out
def relative_rmse(model1, model2):
"""
Given two model objects, compare the model fits to signal-to-signal
reliability in the root mean square error sense.
Parameters
----------
model1, model2: two objects from a class inherited from BaseModel
Returns
-------
relative_RMSE: A measure of goodness of fit, relative to measurement
reliability. The measure is larger than 1 when the model is worse than
signal-to-signal reliability and smaller than 1 when the model is better.
Notes
-----
More specificially, we calculate the rmse from the fit in model1 to the
signal in model2 and then vice-versa. We average between the two
results. Then, we calculate the rmse from the signal in model1 to the
signal in model2. We normalize the average model-to-signal rmse to the
signal-to-signal rmse as a measure of goodness of fit of the model.
"""
# Assume that the last dimension is the signal dimension, so the dimension
# across which the rmse will be calculated:
out = ozu.nans(model1.shape[:-1])
sig1 = model1.signal[model1.mask]
sig2 = model2.signal[model2.mask]
fit1 = model1.fit[model1.mask]
fit2 = model2.fit[model2.mask]
signal_rmse = ozu.rmse(sig1, sig2)
fit1_rmse = ozu.rmse(fit1, sig2)
fit2_rmse = ozu.rmse(fit2, sig1)
# Average in each element:
fit_rmse = (fit1_rmse + fit2_rmse) / 2.
rel_rmse = fit_rmse / signal_rmse
out[model1.mask] = rel_rmse
return out
def relative_rmse(model1, model2):
"""
Given two model objects, compare the model fits to signal-to-signal
reliability in the root mean square error sense.
Parameters
----------
model1, model2: two objects from a class inherited from BaseModel
Returns
-------
relative_RMSE: A measure of goodness of fit, relative to measurement
reliability. The measure is larger than 1 when the model is worse than
signal-to-signal reliability and smaller than 1 when the model is better.
Notes
-----
More specificially, we calculate the rmse from the fit in model1 to the
signal in model2 and then vice-versa. We average between the two
results. Then, we calculate the rmse from the signal in model1 to the
signal in model2. We normalize the average model-to-signal rmse to the
signal-to-signal rmse as a measure of goodness of fit of the model.
"""
# Assume that the last dimension is the signal dimension, so the dimension
# across which the rmse will be calculated:
out = ozu.nans(model1.shape[:-1])
sig1 = model1.signal[model1.mask]
sig2 = model2.signal[model2.mask]
fit1 = model1.fit[model1.mask]
fit2 = model2.fit[model2.mask]
signal_rmse = ozu.rmse(sig1, sig2)
fit1_rmse = ozu.rmse(fit1, sig2)
fit2_rmse = ozu.rmse(fit2, sig1)
# Average in each element:
fit_rmse = (fit1_rmse + fit2_rmse) / 2.
rel_rmse = fit_rmse/signal_rmse
out[model1.mask] = rel_rmse
return out
def test_rms_rmse():
"""
Test the rms and rmse functions
"""
# They both give the same answer:
data = np.random.randn(10)
npt.assert_equal(ozu.rms(data), ozu.rmse(data, np.zeros(data.shape)))
def test_rms_rmse():
"""
Test the rms and rmse functions
"""
# They both give the same answer:
data = np.random.randn(10)
npt.assert_equal(ozu.rms(data), ozu.rmse(data, np.zeros(data.shape)))
def RMSE(self):
"""
We need to overload this to make the shapes to broadcast into make
sense. XXX Need to consider whether it makes sense to take out our
overloaded signal and relative_signal above, so we might not need this
either...
"""
out = ozu.nans(self.signal.shape[:3])
flat_fit = self.fit[self.mask][:,:self.fit.shape[-1]-1]
flat_rmse = ozu.rmse(self._flat_signal, flat_fit)
out[self.mask] = flat_rmse
return out
def RMSE(self):
"""
We need to overload this to make the shapes to broadcast into make
sense. XXX Need to consider whether it makes sense to take out our
overloaded signal and relative_signal above, so we might not need this
either...
"""
out = ozu.nans(self.signal.shape[:3])
flat_fit = self.fit[self.mask][:, :self.fit.shape[-1] - 1]
flat_rmse = ozu.rmse(self._flat_signal, flat_fit)
out[self.mask] = flat_rmse
return out
def rmse(model1, model2):
"""
Calculate the voxel-wise RMSE between one model signal and the other model
signal.
"""
out = ozu.nans(model1.shape[:-1])
sig1 = model1.signal[model1.mask]
sig2 = model2.signal[model2.mask]
out_flat = np.empty(sig1.shape[0])
for vox in xrange(sig1.shape[0]):
out_flat[vox] = ozu.rmse(sig1[vox], sig2[vox])
out[model1.mask] = out_flat
return out
def rmse(model1, model2):
"""
Calculate the voxel-wise RMSE between one model signal and the other model
signal.
"""
out = ozu.nans(model1.shape[:-1])
sig1 = model1.signal[model1.mask]
sig2 = model2.signal[model2.mask]
out_flat = np.empty(sig1.shape[0])
for vox in xrange(sig1.shape[0]):
out_flat[vox] = ozu.rmse(sig1[vox], sig2[vox])
out[model1.mask] = out_flat
return out
def relative_signal_rmse(self, DWI2):
"""
Calculate the voxel-by-voxel RMSE between the relative signal of two
measurements
Parameters
----------
DWI2 : a second DWI class instance.
"""
flat_sig1 = self._flat_relative_signal
flat_sig2 = DWI2._flat_relative_signal
rmse = np.empty(flat_sig2.shape[0])
for vox in xrange(flat_sig1.shape[0]):
rmse[vox] = ozu.rmse(flat_sig1[vox], flat_sig2[vox])
# Re-package it into a volume:
out = ozu.nans(self.shape[:3])
out[self.mask] = rmse
return out
def relative_signal_rmse(self, DWI2):
"""
Calculate the voxel-by-voxel RMSE between the relative signal of two
measurements
Parameters
----------
DWI2 : a second DWI class instance.
"""
flat_sig1 = self._flat_relative_signal
flat_sig2 = DWI2._flat_relative_signal
rmse = np.empty(flat_sig2.shape[0])
for vox in xrange(flat_sig1.shape[0]):
rmse[vox] = ozu.rmse(flat_sig1[vox], flat_sig2[vox])
# Re-package it into a volume:
out = ozu.nans(self.shape[:3])
out[self.mask] = rmse
return out
fractional_anisotropy = (
(Model1.model_params[:, :, :, 1] - Model1.model_params[:, :, :, 2]) /
(Model1.model_params[:, :, :, 1] + Model1.model_params[:, :, :, 2]))
fig = viz.mosaic(T1.T, cmap=matplotlib.cm.bone, cbar=False)
fig = viz.mosaic(fractional_anisotropy.T,
fig=fig,
cmap=matplotlib.cm.RdBu_r,
vmax=1,
vmin=-1)
fig.set_size_inches([12, 8])
fig.savefig(
'/Users/arokem/Dropbox/DWI_figures/MultiCanonicalTensor_FA%s.png' % b)
signal_rmse = np.nan * np.ones(Model1.shape[:3])
signal_rmse[Model1.mask] = ozu.rmse(Model1._flat_signal,
Model2._flat_signal)
model_rmse = np.nan * np.ones(Model1.shape[:3])
model_rmse[Model1.mask] = ozu.rmse(Model1.fit[Model1.mask],
Model2._flat_signal)
relative_rmse = np.nan * np.ones(Model1.shape[:3])
relative_rmse = model_rmse / signal_rmse
relative_rmse[np.isinf(relative_rmse)] = np.nan
ax = viz.quick_ax()
ax.hist(relative_rmse[~np.isnan(relative_rmse)],
histtype='step',
bins=100,
color='g')
fig = ax.get_figure()
fig.set_size_inches([10, 5])
ax.set_xlim([0, 2])
ax.set_ylabel('# voxels')
fig.set_size_inches([12,8])
fig.savefig('/home/arokem/Dropbox/DWI_figures/CanonicalTensor_sphere_param%s.png'%b)
fractional_anisotropy = (
(Model1.model_params[:,:,:,1]-Model1.model_params[:,:,:,2])/
(Model1.model_params[:,:,:,1] + Model1.model_params[:,:,:,2]))
fig = viz.mosaic(T1.T, cmap=matplotlib.cm.bone, cbar=False)
fig = viz.mosaic(fractional_anisotropy.T,fig=fig,
cmap=matplotlib.cm.RdBu_r, vmax=1, vmin=-1)
fig.set_size_inches([12,8])
fig.savefig('/home/arokem/Dropbox/DWI_figures/CanonicalTensor_FA%s.png'%b)
signal_rmse = np.nan * np.ones(Model1.shape[:3])
signal_rmse[Model1.mask] = ozu.rmse(Model1._flat_signal,
Model2._flat_signal)
model_rmse = np.nan * np.ones(Model1.shape[:3])
model_rmse[Model1.mask] = ozu.rmse(Model1.fit[Model1.mask],
Model2._flat_signal)
relative_rmse = np.nan * np.ones(Model1.shape[:3])
relative_rmse = model_rmse/signal_rmse
relative_rmse[np.isinf(relative_rmse)] = np.nan
ax = viz.quick_ax()
ax.hist(relative_rmse[~np.isnan(relative_rmse)],
histtype='step',
bins=100,
color='g')
fig = ax.get_figure()
fig.set_size_inches([10,5])
ax.set_xlim([0,2])
ax.set_ylabel('# voxels')