def test_syntheticFID(): testFID, hdr = syn.syntheticFID(noisecovariance=[[0.0]], points=16384) # Check FID is sum of lorentzian lineshapes # anlytical solution T2 = 1 / (hdr['inputopts']['damping'][0]) M0 = hdr['inputopts']['amplitude'][0] f0 = hdr['inputopts']['centralfrequency'] * hdr['inputopts'][ 'chemicalshift'][0] f1 = hdr['inputopts']['centralfrequency'] * hdr['inputopts'][ 'chemicalshift'][1] f = hdr['faxis'] spec = (M0 * T2) / (1 + 4 * np.pi**2 * (f0 - f)**2 * T2**2) + 1j * ( 2 * np.pi * M0 * (f0 - f) * T2**2) / (1 + 4 * np.pi**2 * (f0 - f)**2 * T2**2) spec += (M0 * T2) / (1 + 4 * np.pi**2 * (f1 - f)**2 * T2**2) + 1j * ( 2 * np.pi * M0 * (f1 - f) * T2**2) / (1 + 4 * np.pi**2 * (f1 - f)**2 * T2**2) # Can't quite get the scaling right here. testSpec = FIDToSpec(testFID[0]) spec /= np.max(np.abs(spec)) testSpec /= np.max(np.abs(testSpec)) assert np.isclose(spec, FIDToSpec(testFID[0]), atol=1E-2, rtol=1E0).all()
def identifyUnlikeFIDs(FIDList,bandwidth,centralFrequency,sdlimit = 1.96,iterations=2,ppmlim=None,shift=True): """ Identify FIDs in a list that are unlike the others Args: FIDList (list of ndarray): Time domain data bandwidth (float) : Bandwidth in Hz centralFrequency (float) : Central frequency in Hz sdlimit (float,optional) : Exclusion limit (number of stnadard deviations). Default = 3. iterations (int,optional): Number of iterations to use. ppmlim (tuple,optional) : Limit to this ppm range shift (bool,optional) : Apply H20 shft Returns: goodFIDS (list of ndarray): FIDs that passed the criteria badFIDS (list of ndarray): FIDs that failed the likeness critera rmIndicies (list of int): Indicies of those FIDs that have been removed metric (list of floats): Likeness metric of each FID """ # Calculate the FID to compare to target = get_target_FID(FIDList,target='median') if ppmlim is not None: MRSargs = {'FID':target,'bw':bandwidth,'cf':centralFrequency} mrs = MRS(**MRSargs) target = extract_spectrum(mrs,target,ppmlim=ppmlim,shift=shift) compareList = [extract_spectrum(mrs,f,ppmlim=ppmlim,shift=shift) for f in FIDList] else: compareList = [FIDToSpec(f) for f in FIDList] target = FIDToSpec(target) # Do the comparison for idx in range(iterations): metric = [] for data in compareList: metric.append(np.linalg.norm(data-target)) metric = np.asarray(metric) metric_avg = np.mean(metric) metric_std = np.std(metric) goodFIDs,badFIDs,rmIndicies,keepIndicies = [],[],[],[] for iDx,(data,m) in enumerate(zip(FIDList,metric)): if m > ((sdlimit*metric_std)+metric_avg) or m < (-(sdlimit*metric_std)+metric_avg): badFIDs.append(data) rmIndicies.append(iDx) else: goodFIDs.append(data) keepIndicies.append(iDx) target = get_target_FID(goodFIDs,target='median') if ppmlim is not None: target = extract_spectrum(mrs,target,ppmlim=ppmlim,shift=shift) else: target = FIDToSpec(target) return goodFIDs,badFIDs,keepIndicies,rmIndicies,metric.tolist()
def FSLModel_grad_vb(x, nu, t, m, B, G, g, first, last): n = m.shape[1] # get number of basis functions logcon, loggamma, eps, phi0, phi1, b = FSLModel_x2param(x, n, g) con = np.exp(logcon) gamma = np.exp(loggamma) # Start E = np.zeros((m.shape[0], g), dtype=np.complex) for gg in range(g): E[:, gg] = np.exp(-(1j * eps[gg] + gamma[gg]) * t).flatten() e_term = np.zeros(m.shape, dtype=np.complex) c = np.zeros((con.size, g)) for i, gg in enumerate(G): e_term[:, i] = E[:, gg] c[i, gg] = con[i] m_term = m * e_term phi_term = np.exp(-1j * (phi0 + phi1 * nu)) Fmet = FIDToSpec(m_term) Ftmet = FIDToSpec(t * m_term) Ftmetc = Ftmet @ c Fmetcon = Fmet @ con[:, None] # Forward model S = (phi_term * Fmetcon) if B is not None: S += B @ b[:, None] # Gradients dSdc = phi_term * Fmet dSdgamma = phi_term * (-Ftmetc) dSdeps = phi_term * (-1j * Ftmetc) dSdphi0 = -1j * phi_term * (Fmetcon) dSdphi1 = -1j * nu * phi_term * (Fmetcon) dSdb = B # Only compute within a range dSdc = dSdc[first:last, :] dSdgamma = dSdgamma[first:last, :] dSdeps = dSdeps[first:last, :] dSdphi0 = dSdphi0[first:last] dSdphi1 = dSdphi1[first:last] dSdb = dSdb[first:last] dS = np.concatenate((dSdc * con[None, :], dSdgamma * gamma[None, :], dSdeps, dSdphi0, dSdphi1, dSdb), axis=1) dS = np.concatenate((np.real(dS), np.imag(dS)), axis=0) return dS
def FSLModel_forward_vb_voigt(x, nu, t, m, B, G, g, first, last): n = m.shape[1] # get number of basis functions logcon, loggamma, logsigma, eps, phi0, phi1, b = FSLModel_x2param_Voigt( x, n, g) con = np.exp(logcon) gamma = np.exp(loggamma) sigma = np.exp(logsigma) E = np.zeros((m.shape[0], g), dtype=np.complex) for gg in range(g): E[:, gg] = np.exp(-(1j * eps[gg] + gamma[gg] + t * sigma[gg]**2) * t).flatten() tmp = np.zeros(m.shape, dtype=np.complex) for i, gg in enumerate(G): tmp[:, i] = m[:, i] * E[:, gg] M = FIDToSpec(tmp) S = np.exp(-1j * (phi0 + phi1 * nu)) * (M @ con[:, None]) # add baseline if B is not None: S += B @ b[:, None] S = S.flatten()[first:last] return np.concatenate((np.real(S), np.imag(S)))
def FSLModel_forward(x, nu, t, m, B, G, g): """ x = [con[0],...,con[n-1],gamma,eps,phi0,phi1,baselineparams] nu : array-like - frequency axis t : array-like - time axis m : basis time course B : baseline functions G : metabolite groups g : number of metab groups Returns forward prediction in the frequency domain """ n = m.shape[1] # get number of basis functions con, gamma, eps, phi0, phi1, b = FSLModel_x2param(x, n, g) E = np.zeros((m.shape[0], g), dtype=np.complex) for gg in range(g): E[:, gg] = np.exp(-(1j * eps[gg] + gamma[gg]) * t).flatten() # E = np.exp(-(1j*eps+gamma)*t) # THis is actually slower! But maybe more optimisable longterm with numexpr or numba tmp = np.zeros(m.shape, dtype=np.complex) for i, gg in enumerate(G): tmp[:, i] = m[:, i] * E[:, gg] M = FIDToSpec(tmp) S = np.exp(-1j * (phi0 + phi1 * nu)) * (M @ con[:, None]) # add baseline if B is not None: S += B @ b[:, None] return S.flatten()
def FSLModel_forward_Voigt(x, nu, t, m, B, G, g): """ x = [con[0],...,con[n-1],gamma,eps,phi0,phi1,baselineparams] nu : array-like - frequency axis t : array-like - time axis m : basis time course B : baseline functions G : metabolite groups g : number of metab groups Returns forward prediction in the frequency domain """ n = m.shape[1] # get number of basis functions con, gamma, sigma, eps, phi0, phi1, b = FSLModel_x2param_Voigt(x, n, g) E = np.zeros((m.shape[0], g), dtype=np.complex) for gg in range(g): E[:, gg] = np.exp(-(1j * eps[gg] + gamma[gg] + t * sigma[gg]**2) * t).flatten() tmp = np.zeros(m.shape, dtype=np.complex) for i, gg in enumerate(G): tmp[:, i] = m[:, i] * E[:, gg] M = FIDToSpec(tmp, axis=0) S = np.exp(-1j * (phi0 + phi1 * nu)) * (M @ con[:, None]) # add baseline if B is not None: S += B @ b[:, None] return S.flatten()
def loadResults(self, mrs, fitResults): "Load fitting results and calculate some metrics" # Populate data frame if fitResults.ndim == 1: self.fitResults = pd.DataFrame(data=fitResults[np.newaxis, :], columns=self.params_names) else: self.fitResults = pd.DataFrame(data=fitResults, columns=self.params_names) self.params = self.fitResults.mean().values #Store prediction, baseline, residual self.pred = self.predictedFID(mrs, mode='Full') self.baseline = self.predictedFID(mrs, mode='Baseline') self.residuals = mrs.FID - self.pred # Calculate single point crlb and cov first, last = mrs.ppmlim_to_range(self.ppmlim) _, _, forward, _, _ = models.getModelFunctions(self.model) forward_lim = lambda p: forward(p, mrs.frequencyAxis, mrs.timeAxis, mrs .basis, self.base_poly, self. metab_groups, self.g)[first:last] data = mrs.getSpectrum(ppmlim=self.ppmlim) # self.crlb = calculate_crlb(self.params,forward_lim,data) self.cov = calculate_lap_cov(self.params, forward_lim, data) self.crlb = np.diagonal(self.cov) std = np.sqrt(self.crlb) self.corr = self.cov / (std[:, np.newaxis] * std[np.newaxis, :]) self.mse = np.mean(np.abs(FIDToSpec(self.residuals)[first:last])**2) with np.errstate(divide='ignore', invalid='ignore'): self.perc_SD = np.sqrt(self.crlb) / self.params * 100 self.perc_SD[self.perc_SD > 999] = 999 # Like LCModel :) self.perc_SD[np.isnan(self.perc_SD)] = 999 # Calculate mcmc metrics if self.method == 'MH': self.mcmc_cov = self.fitResults.cov().values self.mcmc_cor = self.fitResults.corr().values self.mcmc_var = self.fitResults.var().values self.mcmc_samples = self.fitResults.values # VB metrics if self.method == 'VB': self.vb_cov = self.optim_out.cov self.vb_var = self.optim_out.var std = np.sqrt(self.vb_var) self.vb_corr = self.vb_cov / (std[:, np.newaxis] * std[np.newaxis, :]) self.hzperppm = mrs.centralFrequency / 1E6 # Calculate QC metrics self.FWHM, self.SNR = qc.calcQC(mrs, self, ppmlim=(0.2, 4.2)) # Run relative concentration scaling to tCr in 'default' 1H MRS case. Create combined metab at same time to avoid later errors. if (('Cr' in self.metabs) and ('PCr' in self.metabs)): self.combine([['Cr', 'PCr']]) self.calculateConcScaling(mrs)
def calculate_area(mrs, FID, ppmlim=None): """ Calculate area of the real part of the spectrum between two limits """ Spec = FIDToSpec(FID, axis=0) if ppmlim is not None: first, last = mrs.ppmlim_to_range(ppmlim) Spec = Spec[first:last] area = np.trapz(np.real(Spec), axis=0) return area
def test_freqshift(): testFID, testHdrs = syn.syntheticFID(amplitude=[0.0, 1.0 ]) # Single peak at 3 ppm # Shift to 0 ppm dt = 1 / testHdrs['inputopts']['bandwidth'] shift = testHdrs['inputopts']['centralfrequency'] * -3.0 shiftedFID = preproc.freqshift(testFID[0], dt, shift) maxindex = np.argmax(np.abs(FIDToSpec(shiftedFID))) freqOfMax = testHdrs['faxis'][maxindex] assert freqOfMax < 5 and freqOfMax > -5
def calcQC(mrs, res, ppmlim=(0.2, 4.2)): """ Calculate SNR and FWHM on fitted data """ if res.method == 'MH': MCMCUsed = True else: MCMCUsed = False try: if MCMCUsed: # Loop over the individual MH results fwhm = [] snrPeaks = [] for _, rp in res.fitResults.iterrows(): qcres = calcQCOnResults(mrs, res, rp, ppmlim) snrPeaks.append(qcres[0]) fwhm.append(qcres[1]) snrSpec = qcres[2] fwhm = np.asarray(fwhm).T snrPeaks = np.asarray(snrPeaks).T else: # Pass the single Newton results snrPeaks, fwhm, snrSpec = calcQCOnResults(mrs, res, res.params, ppmlim) fwhm = np.asarray(fwhm) snrPeaks = np.asarray(snrPeaks) except NoiseNotFoundError: outShape = (len(res.metabs), res.fitResults.shape[0]) fwhm = np.full(outShape, np.nan) snrSpec = np.nan snrPeaks = np.full(outShape, np.nan) # Calculate the LCModel style SNR based on peak height over SD of residual first, last = mrs.ppmlim_to_range(ppmlim=res.ppmlim) baseline = FIDToSpec(res.predictedFID(mrs, mode='baseline'))[first:last] spectrumMinusBaseline = mrs.getSpectrum(ppmlim=res.ppmlim) - baseline snrResidual_height = np.max(np.real(spectrumMinusBaseline)) rmse = 2.0 * np.sqrt(res.mse) snrResidual = snrResidual_height / rmse # Assemble outputs # SNR output snrdf = pd.DataFrame() for m, snr in zip(res.metabs, snrPeaks): snrdf[f'SNR_{m}'] = pd.Series(snr) SNRobj = SNR(spectrum=snrSpec, peaks=snrdf, residual=snrResidual) fwhmdf = pd.DataFrame() for m, width in zip(res.metabs, fwhm): fwhmdf[f'fwhm_{m}'] = pd.Series(width) return fwhmdf, SNRobj
def test_phase_freq_align(): peak1Shift = np.random.rand(10) * 0.1 peak1Phs = np.random.randn(10) * 2 * np.pi shiftedFIDs = [] for s, p in zip(peak1Shift, peak1Phs): testFIDs, testHdrs = syn.syntheticFID(amplitude=[1, 1], chemicalshift=[-2 + s, 3], phase=[p, 0.0], points=2048, noisecovariance=[[1E-1]]) shiftedFIDs.append(testFIDs[0]) # Align across shifted peak alignedFIDs, _, _ = preproc.phase_freq_align(shiftedFIDs, testHdrs['bandwidth'], testHdrs['centralFrequency'] * 1E6, niter=2, verbose=False, ppmlim=(-2.2, -1.7), shift=False) meanFID = preproc.combine_FIDs(alignedFIDs, 'mean') assert np.max(np.abs(FIDToSpec(meanFID))) > 0.09 # Align across fixed peak alignedFIDs, _, _ = preproc.phase_freq_align(shiftedFIDs, testHdrs['bandwidth'], testHdrs['centralFrequency'] * 1E6, niter=2, verbose=False, ppmlim=(2, 4), shift=False) meanFID = preproc.combine_FIDs(alignedFIDs, 'mean') assert np.max(np.abs(FIDToSpec(meanFID))) > 0.09
def plot_fit_new(mrs, ppmlim=(0.40, 4.2)): """ plot model fitting plus baseline mrs : MRS object ppmlim : tuple """ axis = np.flipud(mrs.ppmAxisFlip) spec = np.flipud(np.fft.fftshift(mrs.Spec)) pred = FIDToSpec(mrs.pred) pred = np.flipud(np.fft.fftshift(pred)) if mrs.baseline is not None: B = np.flipud(np.fft.fftshift(mrs.baseline)) first = np.argmin(np.abs(axis - ppmlim[0])) last = np.argmin(np.abs(axis - ppmlim[1])) if first > last: first, last = last, first freq = axis[first:last] plt.figure(figsize=(9, 10)) plt.plot(axis[first:last], spec[first:last]) plt.gca().invert_xaxis() plt.plot(axis[first:last], pred[first:last], 'r') if mrs.baseline is not None: plt.plot(axis[first:last], B[first:last], 'k') # style stuff plt.minorticks_on() plt.grid(b=True, axis='x', which='major', color='k', linestyle='--', linewidth=.3) plt.grid(b=True, axis='x', which='minor', color='k', linestyle=':', linewidth=.3) return plt.gcf()
def FSLModel_transform_basis(x, nu, t, m, G, g): """ Transform basis by applying frequency shifting/blurring """ n = m.shape[1] # get number of basis functions con, gamma, eps, phi0, phi1, b = FSLModel_x2param(x, n, g) E = np.zeros((m.shape[0], g), dtype=np.complex) for gg in range(g): E[:, gg] = np.exp(-(1j * eps[gg] + gamma[gg]) * t).flatten() tmp = np.zeros(m.shape, dtype=np.complex) for i, gg in enumerate(G): tmp[:, i] = m[:, i] * E[:, gg] M = FIDToSpec(tmp) return SpecToFID(np.exp(-1j * (phi0 + phi1 * nu)) * M)
def specApodise(mrs, amount): """ Apply apodisation to spectrum""" FIDApod = mrs.FID * np.exp(-amount * mrs.getAxes('time')) return FIDToSpec(FIDApod)
def syntheticFromBasis(basis, basis_bandwidth, concentrations, broadening=(9.0, 0.0), shifting=0.0, baseline=[0, 0], coilamps=[1.0], coilphase=[0.0], noisecovariance=[[0.1]], bandwidth=4000.0, points=2048): """ Create synthetic spectra from basis FIDs. Use syntheticFromBasisFile interface.""" # sort out inputs numMetabs = basis.shape[1] if len(concentrations) != numMetabs: raise ValueError('Provide a concentration for each basis spectrum.') if isinstance(broadening, list): if len(broadening) != numMetabs: raise ValueError( 'Broadening values must be either a single tuple or list of tuples with the same number of elements as basis sets.' ) gammas = [b[0] for b in broadening] sigmas = [b[1] for b in broadening] elif isinstance(broadening, tuple): gammas = [broadening[0]] * numMetabs sigmas = [broadening[1]] * numMetabs else: raise ValueError( 'Broadening values must be either a single tuple or list of tuples with the same number of elements as basis sets.' ) if isinstance(shifting, list): if len(shifting) != numMetabs: raise ValueError( 'shifting values must be either a float or list with the same number of elements as basis sets.' ) eps = shifting elif isinstance(shifting, float): eps = [shifting] * numMetabs else: raise ValueError( 'shifting values must be either a float or list with the same number of elements as basis sets.' ) # Form noise vectors ncoils = len(coilamps) noisecovariance = np.asarray(noisecovariance) if len(coilphase) != ncoils: raise ValueError('Length of coilamps and coilphase must match.') if noisecovariance.shape != (ncoils, ncoils): raise ValueError('noisecovariance must be ncoils x ncoils.') noise = np.random.multivariate_normal( np.zeros((ncoils)), noisecovariance, points) + 1j * np.random.multivariate_normal(np.zeros( (ncoils)), noisecovariance, points) # Interpolate basis dwelltime = 1 / bandwidth basis_dwelltime = 1 / basis_bandwidth basis = ts_to_ts(basis, basis_dwelltime, dwelltime, points) # basis = rescale_FID(basis,scale=100) # Create the spectrum baseFID = np.zeros((points), np.complex128) dwellTime = 1 / bandwidth timeAxis = np.linspace(dwellTime, dwellTime * points, points) for b, c, e, g, s in zip(basis.T, concentrations, eps, gammas, sigmas): tmp = b * np.exp(-(1j * e + g + timeAxis * s**2) * timeAxis) M = FIDToSpec(tmp) baseFID += SpecToFID(M * c) # Add baseline # TO DO # Add noise and write tot output list FIDs = [] for cDx, (camp, cphs) in enumerate(zip(coilamps, coilphase)): FIDs.append((camp * np.exp(1j * cphs) * baseFID) + noise[:, cDx]) FIDs = np.asarray(FIDs).T FIDs = np.squeeze(FIDs) return FIDs
def FID2Spec(x): """ Turn FID to spectrum for plotting """ x = FIDToSpec(x) return x
def plot_spectrum(mrs, ppmlim=(0.0, 4.5), FID=None, proj='real', c='k'): """ Plotting the spectrum ---------- FID : array-like bandwidth : float (unit = Hz) centralFrequency : float (unit = Hz) ppmlim : tuple (MIN,MAX) proj : string one of 'real', 'imag', 'abs', or 'angle' """ ppmAxisShift = mrs.getAxes(ppmlim=ppmlim) def axes_style(plt, ppmlim, label=None, xticks=None): plt.xlim(ppmlim) plt.gca().invert_xaxis() plt.xlabel(label) plt.gca().set_xticks(xticks) plt.minorticks_on() plt.grid(b=True, axis='x', which='major', color='k', linestyle='--', linewidth=.3) plt.grid(b=True, axis='x', which='minor', color='k', linestyle=':', linewidth=.3) def doPlot(data, c='b', linewidth=1, linestyle='-', xticks=None): plt.plot(ppmAxisShift, data, color=c, linewidth=linewidth, linestyle=linestyle) axes_style(plt, ppmlim, label='Chemical shift (ppm)', xticks=xticks) # Prepare data for plotting if FID is not None: f, l = mrs.ppmlim_to_range(ppmlim) data = FIDToSpec(FID)[f:l] else: data = mrs.getSpectrum(ppmlim=ppmlim) #m = min(np.real(data)) #M = max(np.real(data)) #ylim = (m-np.abs(M)/10,M+np.abs(M)/10) #plt.ylim(ylim) # Create the figure #plt.figure(figsize=(7,7)) # Some nicer x ticks on the plots if np.abs(ppmlim[1] - ppmlim[0]) > 2: xticks = np.arange(np.ceil(ppmlim[0]), np.floor(ppmlim[1]) + 0.1, 1.0) else: xticks = np.arange(np.around(ppmlim[0], 1), np.around(ppmlim[1], 1) + 0.01, 0.1) exec("doPlot(np.{}(data),c='{}' ,linewidth=2,xticks=xticks)".format( proj, c)) plt.tight_layout() return plt.gcf()
def FSLModel_grad_Voigt(x, nu, t, m, B, G, g, data, first, last): """ x = [con[0],...,con[n-1],gamma,eps,phi0,phi1,baselineparams] nu : array-like - frequency axis t : array-like - time axis m : basis time course B : baseline functions G : metabolite groups g : number of metab groups data : array like - frequency domain data first,last : range for the fitting is data[first:last] returns gradient vector """ n = m.shape[1] # get number of basis functions #g = max(G)+1 # get number of metabolite groups con, gamma, sigma, eps, phi0, phi1, b = FSLModel_x2param_Voigt(x, n, g) # Start E = np.zeros((m.shape[0], g), dtype=np.complex) SIG = np.zeros((m.shape[0], g), dtype=np.complex) for gg in range(g): E[:, gg] = np.exp(-(1j * eps[gg] + gamma[gg] + t * sigma[gg]**2) * t).flatten() SIG[:, gg] = sigma[gg] e_term = np.zeros(m.shape, dtype=np.complex) sig_term = np.zeros(m.shape, dtype=np.complex) c = np.zeros((con.size, g)) for i, gg in enumerate(G): e_term[:, i] = E[:, gg] sig_term[:, i] = SIG[:, gg] c[i, gg] = con[i] m_term = m * e_term phi_term = np.exp(-1j * (phi0 + phi1 * nu)) Fmet = FIDToSpec(m_term) Ftmet = FIDToSpec(t * m_term) Ft2sigmet = FIDToSpec(t * t * sig_term * m_term) Ftmetc = Ftmet @ c Ft2sigmetc = Ft2sigmet @ c Fmetcon = Fmet @ con[:, None] Spec = data[first:last, None] # Forward model S = (phi_term * Fmetcon) if B is not None: S += B @ b[:, None] # Gradients dSdc = phi_term * Fmet dSdgamma = phi_term * (-Ftmetc) dSdsigma = phi_term * (-2 * Ft2sigmetc) dSdeps = phi_term * (-1j * Ftmetc) dSdphi0 = -1j * phi_term * (Fmetcon) dSdphi1 = -1j * nu * phi_term * (Fmetcon) dSdb = B # Only compute within a range S = S[first:last] dSdc = dSdc[first:last, :] dSdgamma = dSdgamma[first:last, :] dSdsigma = dSdsigma[first:last, :] dSdeps = dSdeps[first:last, :] dSdphi0 = dSdphi0[first:last] dSdphi1 = dSdphi1[first:last] dSdb = dSdb[first:last] dS = np.concatenate( (dSdc, dSdgamma, dSdsigma, dSdeps, dSdphi0, dSdphi1, dSdb), axis=1) grad = np.real( np.sum(S * np.conj(dS) + np.conj(S) * dS - np.conj(Spec) * dS - Spec * np.conj(dS), axis=0)) return grad