def test_example_2_Gaussians_1_exp():
dat = np.loadtxt('NIST_Gauss2.dat')
x = dat[:, 1]
y = dat[:, 0]
exp_mod = ExponentialModel(prefix='exp_')
pars = exp_mod.guess(y, x=x)
gauss1 = GaussianModel(prefix='g1_')
pars.update(gauss1.make_params())
pars['g1_center'].set(105, min=75, max=125)
pars['g1_sigma'].set(15, min=3)
pars['g1_amplitude'].set(2000, min=10)
gauss2 = GaussianModel(prefix='g2_')
pars.update(gauss2.make_params())
pars['g2_center'].set(155, min=125, max=175)
pars['g2_sigma'].set(15, min=3)
pars['g2_amplitude'].set(2000, min=10)
mod = gauss1 + gauss2 + exp_mod
init = mod.eval(pars, x=x)
plt.plot(x, y)
plt.plot(x, init, 'k--')
out = mod.fit(y, pars, x=x)
print(out.fit_report(min_correl=0.5))
plt.plot(x, out.best_fit, 'r-')
plt.show()
def predictive_model(data: pd.DataFrame,
interesting_rows,
day_zero_n_patients: int = 20,
days_in_future: int = 30,
aggregated: bool = False):
data = data[interesting_rows].iloc[:, :]
from lmfit.models import StepModel, ExponentialModel
fig = plt.figure(figsize=(10, 5))
for c in range(len(data.index)):
if aggregated:
values = data.values[c, 4:][data.iloc[c, 4:] > day_zero_n_patients]
else:
values = np.concatenate(
([0],
np.diff(
data.values[c,
4:][data.iloc[c, 4:] > day_zero_n_patients])))
n = values.shape[0]
x = np.asarray(range(values.shape[0]), dtype='float64')
y = np.asarray(values, dtype='float64')
if len(x) == 0:
continue
label = "{}-{}".format(data.values[c, 0], data.values[c, 1])
plt.plot(x, y, label=label)
if data.values[c, 1] in ["China", "US"]:
continue
try:
model_step = StepModel()
model_exp = ExponentialModel()
params_step = model_step.guess(y, x=x)
params_exp = model_exp.guess(y, x=x)
result_step = model_step.fit(y, params_step, x=x)
result_exp = model_exp.fit(y, params_exp, x=x)
except Exception:
continue
x_pred = np.asarray(range(days_in_future))
plt.plot(x_pred,
model_step.eval(result_step.params, x=x_pred),
':',
label='fit-{}'.format(label))
plt.plot(x_pred,
model_exp.eval(result_exp.params, x=x_pred),
'.',
label='fit-{}'.format(label))
# print(result.fit_report())
# result.plot_fit()
plt.legend(prop={"size": 7})
plt.yscale('log')
plt.xticks(rotation=45)
plt.grid(which='both')
now = datetime.now()
dt_string = now.strftime("%d%m%Y-%H%M%S")
def exponential_fit(x, y, errors=True):
from lmfit.models import ExponentialModel
mod = ExponentialModel()
par = mod.guess(data=y, x=x)
out = mod.fit(y, params=par, x=x)
a = out.params['amplitude']
d = out.params['decay']
return a, d
def exponential_fit(x, y, errors=True):
from lmfit.models import ExponentialModel
mod = ExponentialModel()
par = mod.guess(data=y, x=x)
out = mod.fit(y, params=par, x=x)
a = out.params["amplitude"]
d = out.params["decay"]
return a, d
def predict_detector_position(datatable=None, target=20500):
dt = array(datatable)
mod = ExponentialModel()
prediction = []
for i in range(1, 5):
signal = dt[:, i]
pars = mod.guess(signal, x=dt[:, 0])
out = mod.fit(signal, pars, x=dt[:, 0])
#out.plot()
#print(whisper(out.fit_report(min_correl=0)))
amp = out.params['amplitude']
tau = out.params['decay']
prediction.append(ceil(tau * log(amp / target)))
#print(go_msg(f'predictions are {prediction}'))
return prediction
def test_numdifftools_no_bounds(fit_method):
pytest.importorskip("numdifftools")
np.random.seed(7)
x = np.linspace(0, 100, num=50)
noise = np.random.normal(scale=0.25, size=x.size)
y = exponential(x, amplitude=5, decay=15) + noise
mod = ExponentialModel()
params = mod.guess(y, x=x)
# do fit, here with leastsq model
result = mod.fit(y, params, x=x, method='leastsq')
result_ndt = mod.fit(y, params, x=x, method=fit_method)
# assert that fit converged to the same result
vals = [result.params[p].value for p in result.params.valuesdict()]
vals_ndt = [
result_ndt.params[p].value for p in result_ndt.params.valuesdict()
]
assert_allclose(vals_ndt, vals, rtol=0.1)
assert_allclose(result_ndt.chisqr, result.chisqr, rtol=1e-5)
# assert that parameter uncertaintes from leastsq and calculated from
# the covariance matrix using numdifftools are very similar
stderr = [result.params[p].stderr for p in result.params.valuesdict()]
stderr_ndt = [
result_ndt.params[p].stderr for p in result_ndt.params.valuesdict()
]
perr = np.array(stderr) / np.array(vals)
perr_ndt = np.array(stderr_ndt) / np.array(vals_ndt)
assert_almost_equal(perr_ndt, perr, decimal=3)
# assert that parameter correlatations from leastsq and calculated from
# the covariance matrix using numdifftools are very similar
for par1 in result.var_names:
cor = [
result.params[par1].correl[par2]
for par2 in result.params[par1].correl.keys()
]
cor_ndt = [
result_ndt.params[par1].correl[par2]
for par2 in result_ndt.params[par1].correl.keys()
]
assert_almost_equal(cor_ndt, cor, decimal=2)
def create_model_params(x, y):
exp_mod = ExponentialModel(prefix='exp_')
params = exp_mod.guess(y, x=x)
gauss1 = GaussianModel(prefix='g1_')
params.update(gauss1.make_params())
gauss2 = GaussianModel(prefix='g2_')
params.update(gauss2.make_params())
params['g1_center'].set(105, min=75, max=125)
params['g1_sigma'].set(15, min=3)
params['g1_amplitude'].set(2000, min=10)
params['g2_center'].set(155, min=125, max=175)
params['g2_sigma'].set(15, min=3)
params['g2_amplitude'].set(2000, min=10)
model = gauss1 + gauss2 + exp_mod
return model, params
def fit_peak(wl, np_x, np_y, with_plot=True, plot_components=False):
''' Fit one peak
wl wavelength
x axis values as a numpy array
y axis values as a numpy array
'''
i_min = len(np_x)-sum(np_x>wl) - 10
i_max = len(np_x)-sum(np_x>wl) + 10
x = np_x[i_min:i_max]
many_x = np.arange(x[0], x[-1], 0.01)
y = np_y[i_min:i_max]
exp_mod = ExponentialModel(prefix='exp_')
pars = exp_mod.guess(y, x=x)
gauss1 = GaussianModel(prefix='g1_')
pars.update(gauss1.make_params())
pars['g1_center'].set(wl, min=wl-5, max=wl+5)
pars['g1_sigma'].set(1, min=0.01)
pars['g1_amplitude'].set(50000, min=10)
mod = gauss1 + exp_mod
# init = mod.eval(pars, x=many_x)
out = mod.fit(y, pars, x=x)
comps = out.eval_components(x=many_x)
print(out.fit_report(min_correl=0.5))
print('Gaussian line model')
try:
print('Mean={:.2f}+/-{:.2f}\tFWHM={:.2f}+/-{:.2f}'.format(
out.params['g1_center'].value, out.params['g1_center'].stderr,
out.params['g1_fwhm'].value, out.params['g1_fwhm'].stderr))
except:
print('Fit did not converge')
if with_plot:
fig = plt.figure()
out.plot(fig=fig, numpoints=100)
plt.show()
return out
def create_model_params(x, y):
"""Create the model and parameters."""
exp_mod = ExponentialModel(prefix='exp_')
params = exp_mod.guess(y, x=x)
gauss1 = GaussianModel(prefix='g1_')
params.update(gauss1.make_params())
gauss2 = GaussianModel(prefix='g2_')
params.update(gauss2.make_params())
params['g1_center'].set(value=105, min=75, max=125)
params['g1_sigma'].set(value=15, min=3)
params['g1_amplitude'].set(value=2000, min=10)
params['g2_center'].set(value=155, min=125, max=175)
params['g2_sigma'].set(value=15, min=3)
params['g2_amplitude'].set(value=2000, min=10)
model = gauss1 + gauss2 + exp_mod
return model, params
def test_numdifftools_no_bounds(fit_method):
pytest.importorskip("numdifftools")
np.random.seed(7)
x = np.linspace(0, 100, num=50)
noise = np.random.normal(scale=0.25, size=x.size)
y = exponential(x, amplitude=5, decay=15) + noise
mod = ExponentialModel()
params = mod.guess(y, x=x)
# do fit, here with leastsq model
result = mod.fit(y, params, x=x, method='leastsq')
result_ndt = mod.fit(y, params, x=x, method=fit_method)
# assert that fit converged to the same result
vals = [result.params[p].value for p in result.params.valuesdict()]
vals_ndt = [result_ndt.params[p].value for p in result_ndt.params.valuesdict()]
assert_allclose(vals_ndt, vals, rtol=0.1)
assert_allclose(result_ndt.chisqr, result.chisqr, rtol=1e-5)
# assert that parameter uncertaintes from leastsq and calculated from
# the covariance matrix using numdifftools are very similar
stderr = [result.params[p].stderr for p in result.params.valuesdict()]
stderr_ndt = [result_ndt.params[p].stderr for p in result_ndt.params.valuesdict()]
perr = np.array(stderr) / np.array(vals)
perr_ndt = np.array(stderr_ndt) / np.array(vals_ndt)
assert_almost_equal(perr_ndt, perr, decimal=3)
# assert that parameter correlatations from leastsq and calculated from
# the covariance matrix using numdifftools are very similar
for par1 in result.var_names:
cor = [result.params[par1].correl[par2] for par2 in
result.params[par1].correl.keys()]
cor_ndt = [result_ndt.params[par1].correl[par2] for par2 in
result_ndt.params[par1].correl.keys()]
assert_almost_equal(cor_ndt, cor, decimal=2)
def test_example_2_Gaussians_1_exp():
dat = np.loadtxt('NIST_Gauss2.dat')
x = dat[:, 1]
y = dat[:, 0]
exp_mod = ExponentialModel(prefix='exp_')
pars = exp_mod.guess(y, x=x)
gauss1 = GaussianModel(prefix='g1_')
pars.update(gauss1.make_params())
pars['g1_center'].set(105, min=75, max=125)
pars['g1_sigma'].set(15, min=3)
pars['g1_amplitude'].set(2000, min=10)
gauss2 = GaussianModel(prefix='g2_')
pars.update(gauss2.make_params())
pars['g2_center'].set(155, min=125, max=175)
pars['g2_sigma'].set(15, min=3)
pars['g2_amplitude'].set(2000, min=10)
mod = gauss1 + gauss2 + exp_mod
init = mod.eval(pars, x=x)
plt.plot(x, y)
plt.plot(x, init, 'k--')
out = mod.fit(y, pars, x=x)
print(out.fit_report(min_correl=0.5))
plt.plot(x, out.best_fit, 'r-')
plt.show()
exp_mod = ExponentialModel(prefix='exp_')
gauss1 = GaussianModel(prefix='g1_')
gauss2 = GaussianModel(prefix='g2_')
def index_of(arrval, value):
"return index of array *at or below* value "
if value < min(arrval): return 0
return max(np.where(arrval <= value)[0])
ix1 = index_of(x, 75)
ix2 = index_of(x, 135)
ix3 = index_of(x, 175)
pars1 = exp_mod.guess(y[:ix1], x=x[:ix1])
pars2 = gauss1.guess(y[ix1:ix2], x=x[ix1:ix2])
pars3 = gauss2.guess(y[ix2:ix3], x=x[ix2:ix3])
pars = pars1 + pars2 + pars3
mod = gauss1 + gauss2 + exp_mod
out = mod.fit(y, pars, x=x)
print(out.fit_report(min_correl=0.5))
plt.plot(x, y)
plt.plot(x, out.init_fit, 'k--')
plt.plot(x, out.best_fit, 'r-')
plt.show()
#<end examples/doc_nistgauss2.py>
def produce_taufits(filepath,meth='iso',pulseperiod=None,snr_cut=None,
verbose=True, plotparams=False, plotflux=False, savefigure=False):
pulsar, nch, nbins,nsub, lm_rms, tsub = read_headerfull(filepath)
if verbose == True:
verboseTag = True
else:
verboseTag = False
print0 = "Pulsar name: %s" %pulsar
print1 = "Number of freq. channels: %d \nFreq channels will be labeled 0 - %d" %(nch, nch-1)
print2 = "Number of bins: %d" %nbins
print3 = "RMS: %.2f" %lm_rms
print4 = "Tsub: %.2f sec" %tsub
for k in range(5):
print eval('print{0}'.format(k))
print"--------------------------------------------------------"
if pulseperiod==None:
## Define time axis, and time/bins conversions
print ("Using Tsub in header to convert bins to time. Assumption is that tsub corresponds to 1.0 phase, corresponding to nbins. This should be adapted for search data.")
pulseperiod = tsub
else:
pulseperiod = pulseperiod #set to provided pulseperiod in seconds
profilexaxis = np.linspace(0,pulseperiod,nbins)
pbs = pulseperiod/nbins
tbs = tsub/nbins
"""Initialise vector outputs"""
obtainedtaus, lmfittausstds = [], []
"""freqmsMHz will correctly associate scattering time values
(tau) with frequency, taking into account frequency integration
across a sub-band. Whereas freqcsMHz is the centre freq. to the subband"""
freqmsMHz, freqcsMHz = [], []
noiselessmodels =[]
results, datas, comp_SNRs, comp_rmss = [], [], [], []
redchis, paramset, paramset_std, correls = [], [], [], []
halfway = nbins/2.
for i in range(nch):
print"--------------------------------------------------------"
print "Channel %d" %i
"""Read in (pdv) data"""
data, freqc, freqm = read_data(filepath,i,nbins)
freqmsMHz.append(freqm)
freqcsMHz.append(freqc)
# roll the data of lowest freq channel to middle of bins
if i ==0:
peakbin = np.argmax(data)
shift = int(halfway -int(peakbin))
if verboseTag:
print 'peak bin at lowest freq channel:%d' %peakbin
else:
peakbin = peakbin
shift = int(halfway - int(peakbin))
data = np.roll(data,shift)
if verboseTag:
print "Rolling data by -%d bins" %shift
comp_rms = find_rms(data,nbins)
if meth is None:
print "No fitting method was chosen. Will default to an isotropic fitting model. \n Use option -m with 'onedim' to change."
result, noiselessmodel, besttau, taustd, bestparams, bestparams_std, redchi, corsig = tau_fitter(data,nbins,verbose=verboseTag)
elif meth == 'iso':
result, noiselessmodel, besttau, taustd, bestparams, bestparams_std, redchi, corsig = tau_fitter(data,nbins,verbose=verboseTag)
elif meth == 'onedim':
result, noiselessmodel, besttau, taustd, bestparams, bestparams_std, redchi, corsig = tau_1D_fitter(data,nbins)
comp_SNR_model = find_peaksnr(noiselessmodel,comp_rms)
if verboseTag:
print 'Estimated SNR (from model peak and data rms): %.2f' % comp_SNR_model
comp_SNR = find_peaksnr_smooth(data,comp_rms)
print 'Estimated SNR (from data peak and rms): %.2f' % comp_SNR
print 'Channel Tau (ms): %.2f \pm %.2f ms' %(besttau,taustd)
obtainedtaus.append(besttau)
lmfittausstds.append(taustd)
noiselessmodels.append(noiselessmodel)
results.append(result)
datas.append(data)
comp_SNRs.append(comp_SNR)
#new:
comp_rmss.append(comp_rms)
redchis.append(redchi)
paramset.append(bestparams)
paramset_std.append(bestparams_std)
# if plotflux == True:
# correls.append(corsig)
#if plotflux == True:
# cor_sigA = np.zeros(len(correls))
# for i in range(len(correls)):
# cor_sigA[i] = correls[i]['A']
paramset = np.transpose(paramset)
paramset_std = np.transpose(paramset_std)
"""Next check if any of the subbands contain only zeros. This happens with high RFI excision in LOFAR bands"""
zero_ch = []
for i in range(nch):
all_zeros = not np.any(datas[i])
if all_zeros:
zero_ch.append(i)
print"--------------------------------------------------------"
if zero_ch:
print "\n"
print "%d channels have all zeroes (channels(s):" %len(zero_ch), zero_ch, ") and will be removed."
if verboseTag:
print "All zero channels are assigned SNR of 0"
if snr_cut:
print "Using SNR cutoff of %.2f" %snr_cut
comp_SNRs = np.nan_to_num(comp_SNRs)
(ind_lowSNR,) = np.where(np.array(comp_SNRs) < snr_cut)
print "SNR cutoff will exclude %d channels (channel(s): %s)" %(len(ind_lowSNR), ind_lowSNR)
data_highsnr = np.delete(np.array(datas),ind_lowSNR,0)
model_highsnr = np.delete(np.array(noiselessmodels),ind_lowSNR,0)
taus_highsnr = np.delete(np.array(obtainedtaus),ind_lowSNR)
lmfitstds_highsnr = np.delete(np.array(lmfittausstds),ind_lowSNR)
freqMHz_highsnr = np.delete(np.array(freqmsMHz),ind_lowSNR)
#New:
comp_rmss_highsnr = np.delete(np.array(comp_rmss),ind_lowSNR)
redchis_highsnr = np.delete(np.array(redchis),ind_lowSNR)
#corsigA_highsnr = np.delete(np.array(cor_sigA),ind_lowSNR)
paramset_highsnr = np.zeros([len(paramset),len(data_highsnr)])
paramsetstd_highsnr = np.zeros([len(paramset),len(data_highsnr)])
for i in range(len(paramset)):
paramset_highsnr[i]= np.delete(paramset[i],ind_lowSNR)
paramsetstd_highsnr[i]= np.delete(paramset_std[i],ind_lowSNR)
elif (snr_cut == None) and (zero_ch != []):
print "Used no SNR cutoff"
"""Rename array to be same as when cut-off is used"""
"""If no SNR cutoff is used, remove channels with all zeroes
-- these will automatically be removed by any snr_cut > 0"""
data_highsnr = np.delete(np.array(datas),zero_ch,0)
model_highsnr = np.delete(np.array(noiselessmodels),zero_ch,0)
taus_highsnr = np.delete(np.array(obtainedtaus),zero_ch)
lmfitstds_highsnr = np.delete(np.array(lmfittausstds),zero_ch)
freqMHz_highsnr = np.delete(np.array(freqmsMHz),zero_ch)
# New:
comp_rmss_highsnr = np.delete(np.array(comp_rmss),zero_ch)
redchis_highsnr = np.delete(np.array(redchis),zero_ch)
#corsigA_highsnr = np.delete(np.array(cor_sigA),zero_ch)
paramset_highsnr = np.zeros([len(paramset),len(data_highsnr)])
paramsetstd_highsnr = np.zeros([len(paramset),len(data_highsnr)])
for i in range(len(paramset)):
paramset_highsnr[i]= np.delete(paramset[i],zero_ch)
paramsetstd_highsnr[i]= np.delete(paramset_std[i],zero_ch)
else:
print "Used no SNR cutoff and there are no empty channels"
data_highsnr = np.array(datas)
model_highsnr = np.array(noiselessmodels)
taus_highsnr = np.array(obtainedtaus)
lmfitstds_highsnr = np.array(lmfittausstds)
freqMHz_highsnr = np.array(freqmsMHz)
# New:
comp_rmss_highsnr = np.array(comp_rmss)
redchis_highsnr = np.array(redchis)
paramset_highsnr = np.array(paramset)
paramsetstd_highsnr = np.array(paramset_std)
taussec_highsnr = taus_highsnr*pbs
lmfitstdssec_highsnr = lmfitstds_highsnr*pbs
number_of_plotted_channels = len(data_highsnr)
npch = number_of_plotted_channels
print "Will plot remaining %d/%d channels" %(npch, nch)
"""Plotting starts"""
#plot onedim in blue dashed
#else plot in red
if meth == 'onedim':
prof = 'b--'
lcol='b'
else:
prof = 'r-'
lcol ='r'
"""1. PLOT PROFILES"""
dimx, dimy = 3., 3.
numsubplots = dimx*dimy
numplots = int(np.ceil(npch/numsubplots))
print "Num profile plots:", numplots
"""Compute residuals"""
#"""Plot 1: Pulse profiles and fits"""
if npch > 0:
resdata = data_highsnr - model_highsnr
resnormed = (resdata-resdata.mean())/resdata.std()
if taussec_highsnr[0] > 1:
taulabel = taussec_highsnr
taulabelerr = lmfitstdssec_highsnr
taustring = 'sec'
else:
taulabel = taussec_highsnr*1000
taulabelerr = lmfitstdssec_highsnr*1000
taustring = 'ms'
for k in range(numplots):
j = int(numsubplots*k)
figg = plt.figure(k+1,figsize=(int(4*dimx),int(3*dimy)))
plots_remaining = int(npch - numsubplots*k)
#print "Plots remaining", plots_remaining
for i in range(np.min([int(numsubplots),int(plots_remaining)])):
figg.subplots_adjust(left = 0.08, right = 0.98, wspace=0.35,hspace=0.35,bottom=0.15)
#plt.rc('text', usetex=True)
plt.rc('font', family='serif')
plt.subplot(dimx,dimy,i+1)
plt.plot(profilexaxis,data_highsnr[j+i],alpha = 0.20)
plt.plot(profilexaxis,model_highsnr[j+i],lw = 2.0, alpha = 0.85, label=r'$\tau: %.2f \pm%.2f$ %s' %(taulabel[j+i], taulabelerr[j+i], taustring))
plt.title('%s at %.1f MHz' %(pulsar, freqMHz_highsnr[j+i]))
plt.ylim(ymax=1.3*np.max(data_highsnr[j+i]))
plt.xlim(xmax=pulseperiod)
plt.xticks(fontsize=11)
plt.yticks(fontsize=11)
plt.xlabel('time (s)',fontsize=11)
plt.legend(fontsize=11,numpoints=1)
plt.ylabel('normalized intensity',fontsize=11)
plt.tight_layout()
if savefigure == True:
figname = '%s_%s_%s_%d.png' %(os.path.basename(filepath),'fitted_profiles', meth, k)
plt.savefig(figname, dpi=200)
print "Saved figure %s in ./" %figname
if noshow == False:
plt.show()
if verboseTag:
for i in range(npch):
print "Channel %d" %i
print'Tau (ms): %.2f' %(1000*taussec_highsnr[i])
tau1GHz = tauatfreq(freqMHz_highsnr[i]/1000.,taussec_highsnr[i],1.0,4)
print 'tau1GHz_alpha_4 (ms) ~ %.4f' %(tau1GHz*1000)
lmfitstdssec_highsnr = lmfitstdssec_highsnr[np.nonzero(lmfitstdssec_highsnr)]
taussec_highsnr = taussec_highsnr[np.nonzero(lmfitstdssec_highsnr)]
freqMHz_highsnr = freqMHz_highsnr[np.nonzero(lmfitstdssec_highsnr)]
"""Plot 2: Plot Gaussian fitting parameters and DM if selected"""
if plotparams==True:
print "\nPlotting Gaussian fit parameters w.r.t frequency\n"
"""Set plotting parameters"""
alfval = 0.6
markr= '*'
msize=12
plt.figure(numplots+1, figsize=(12,8))
plt.subplots_adjust(left = 0.055, right=0.98,wspace=0.35,hspace=0.4,bottom=0.08)
"""Fit models to sigma"""
powmod = PowerLawModel()
powpars = powmod.guess(paramset_highsnr[0], x=freqMHz_highsnr)
powout = powmod.fit(paramset_highsnr[0], powpars, x=freqMHz_highsnr, weights=1/((paramsetstd_highsnr[0])**2))
linmod = LinearModel()
if len(freqMHz_highsnr) < 3:
raise RuntimeError("plotparams == True: Less than three frequency channels. Cannot compute quadratic or exponential fit for width evolution. Consider lowering snr_cut.")
else:
quadmod = QuadraticModel()
quadpars = quadmod.guess(paramset_highsnr[0], x=freqMHz_highsnr)
quadout = quadmod.fit(paramset_highsnr[0], quadpars, x=freqMHz_highsnr, weights=1/((paramsetstd_highsnr[0])**2))
expmod = ExponentialModel()
exppars = expmod.guess(paramset_highsnr[0], x=freqMHz_highsnr)
expout = expmod.fit(paramset_highsnr[0], exppars, x=freqMHz_highsnr, weights=1/((paramsetstd_highsnr[0])**2))
"""Fit a DM model to delta mu"""
delnuarray = [-(1/freqMHz_highsnr[-1]**2-1/freqMHz_highsnr[i]**2) for i in range(npch)] ##in MHz
delmuarray = [(paramset_highsnr[1][-1] - paramset_highsnr[1][i])*pbs for i in range(npch)] ##in seconds
delmu_stdarray = [(paramsetstd_highsnr[1][-1] - paramsetstd_highsnr[1][i])*pbs for i in range(npch)]
DM_linpars = linmod.guess(delmuarray, x=delnuarray)
DM_linout = linmod.fit(delmuarray, DM_linpars, x=delnuarray)
DM_CCval = DM_linout.best_values['slope']
DM_CCvalstd = DM_linout.params['slope'].stderr
DMmodelfit = DM_linout.best_fit ##model gives deltime in seconds (used to shift data)
DMconstant = 4148.808
#uncertainty in the constant is 0.003 - only affects the Delta DM value in the 9th decimal
DMval = (DM_CCval/DMconstant)
DMvalstd = (DM_CCvalstd/DMconstant)
#DMcheck = psr.DM_checker(freqmsMHz,bestpT_highSNR[1]*pbs)
## Plot reduced chi square:
plt.subplot(2,3,1)
plt.plot(freqMHz_highsnr, redchis_highsnr/np.power(comp_rmss_highsnr,2), markr,alpha=alfval,markersize = msize)
plt.title(r'Reduced $\chi^2$ values', fontsize=12)
plt.yticks(fontsize=12)
plt.xticks(fontsize=12)
plt.xlabel(r'$\nu$ MHz',fontsize=12)
plt.ylabel(r'$\chi^2$',fontsize=12)
## Plot sigma:
plt.subplot(2,3,2)
#plt.errorbar(freqMHz_highsnr,paramset_highsnr[0]*pbs)
plt.errorbar(freqMHz_highsnr,paramset_highsnr[0]*pbs,yerr =paramsetstd_highsnr[0]*pbs, fmt = markr,markersize=msize,capthick=2,linewidth=1.5,alpha=alfval)
plt.plot(freqMHz_highsnr,powout.best_fit*pbs,'-', alpha=alfval,label='pow = %.2f' %powout.best_values['exponent'])
plt.plot(freqMHz_highsnr,quadout.best_fit*pbs,'-',alpha=alfval, label='quad: a,b = %.3f,%.3f' %(quadout.best_values['a'],quadout.best_values['b']))
plt.ylabel(r'$\sigma$ (sec)')
plt.title(r'Width evolution', fontsize=12)
plt.yticks(fontsize=12)
plt.xticks(fontsize=12)
plt.xlabel(r'$\nu$ MHz',fontsize=14)
plt.legend(fontsize = 10, loc='best')
## Plot mean:
plt.subplot(2,3,3)
#plt.errorbar(freqMHz_highsnr,paramset_highsnr[1]*pbs)
plt.errorbar(freqMHz_highsnr,paramset_highsnr[1]*pbs,yerr =paramsetstd_highsnr[1]*pbs, fmt = markr,markersize=msize,capthick=2,linewidth=1.5,alpha=alfval)
plt.ylabel(r'$\mu$ (sec)')
plt.title(r'Centroid evolution', fontsize=12)
plt.yticks(fontsize=12)
plt.xticks(fontsize=12)
plt.xlabel(r'$\nu$ MHz',fontsize=14)
#plt.legend(fontsize = 9, loc='best')
## Plot amplitude:
plt.subplot(2,3,4)
#plt.errorbar(freqMHz_highsnr,paramset_highsnr[2]*pbs)
plt.errorbar(freqMHz_highsnr,paramset_highsnr[2]*pbs,yerr =paramsetstd_highsnr[2]*pbs, fmt = markr,markersize=msize,capthick=2,linewidth=1.5,alpha=alfval)
plt.ylabel(r'$\mu$ (sec)')
plt.title(r'Amplitude evolution', fontsize=12)
plt.yticks(fontsize=12)
plt.xticks(fontsize=12)
plt.xlabel(r'$\nu$ MHz',fontsize=14)
#plt.legend(fontsize = 9, loc='best')
## Plot DC:
plt.subplot(2,3,5)
#plt.errorbar(freqMHz_highsnr,paramset_highsnr[2]*pbs)
plt.errorbar(freqMHz_highsnr,paramset_highsnr[3]*pbs,yerr =paramsetstd_highsnr[3]*pbs, fmt = markr,markersize=msize,capthick=2,linewidth=1.5,alpha=alfval)
plt.ylabel(r'$\mu$ (sec)')
plt.title(r'DC offset', fontsize=12)
plt.yticks(fontsize=12)
plt.xticks(fontsize=12)
plt.xlabel(r'$\nu$ MHz',fontsize=14)
#plt.legend(fontsize = 9, loc='best')
## Plot DM:
plt.subplot(2,3,6)
plt.errorbar(delmuarray,freqMHz_highsnr, fmt = markr, xerr=delmu_stdarray, alpha = alfval, markersize=msize)
plt.plot(DMmodelfit,freqMHz_highsnr, '-', label=r'DM: $%.3f \pm %.3f$ $\rm{pc.cm}^{-3}$' %(DMval,DMvalstd), alpha = alfval)
plt.xlabel(r'$\Delta \mu$ (sec)', fontsize =12)
plt.yticks(fontsize=12)
plt.xticks(fontsize=12)
plt.title('Delta DM', fontsize=12)
plt.ylabel(r'$\nu$ (MHz)',fontsize=14)
plt.ticklabel_format(style='sci', axis='x',scilimits=(0,0))
plt.legend(fontsize = 10, loc='best')
plt.tight_layout()
if savefigure == True:
figname2 = '%s_%s.png' %(os.path.basename(filepath),'fitting_parameters')
plt.savefig(figname2, dpi=200)
print "Saved figure %s in ./" %figname2
if noshow == False:
plt.show()
if plotflux == True: ##Flux section needs debugging
ls = 'solid'
"""Plot flux, and corrected flux spectrum"""
"""Create unscattered profiles, i.e. Guassians"""
bins, profiles = [],[makeprofile(nbins = nbins, ncomps = 1, amps = paramset_highsnr[2][j], means = paramset_highsnr[1][j], sigmas = paramset_highsnr[0][j]) for j in range(npch)]
unscatflux = []
for i in range(npch):
unscatfl = np.sum(profiles[j])/nbins
unscatflux.append(unscatfl)
#smootheddata = smooth(data_highsnr[j],int(0.05*nbins))
scatflux = [find_modelflux(model_highsnr[i],nbins) for i in range(npch)]
climbvals = []
for i in range(npch):
climb = returnclimb(np.linspace(1,nbins,nbins),paramset_highsnr[1][i],paramset_highsnr[0][i],paramset_highsnr[2][i],taussec_highsnr[i],paramset_highsnr[3][i],nbins)
climbvals.append(climb)
correctedflux = np.array([scatflux[i] + climbvals[i] for i in range(npch)])
print scatflux
print climbvals
#per bin
meancorflux = np.mean(correctedflux)
meancorfluxerr = np.sqrt(np.sum(correctedflux**2))/len(correctedflux)
"""Calculate error in Flux"""
sigmaWIDTH = paramsetstd_highsnr[0]*pbs #in seconds
sigmaAMP = paramsetstd_highsnr[2] #in mJy
WIDTHS =paramset_highsnr[0]*pbs #in seconds
AMPS =paramset_highsnr[2] #in mJy
Expr1 = np.sqrt(2*np.pi)*AMPS
Expr2 = np.sqrt(WIDTHS)
AreaExpression = Expr1*Expr2
sigmaEx1 = np.sqrt(2*np.pi)*sigmaAMP
sigmaEx2 = Expr2*0.5*sigmaWIDTH/WIDTHS
sigmaFlux =AreaExpression*np.sqrt(np.power(sigmaEx1/Expr1,2)+ np.power(sigmaEx2/Expr2,2))#+ 2*corsigA_highsnr*sigmaEx1*sigmaEx2/(Expr1*Expr2)
plt.figure(figsize=(10,6))
plt.plot(freqMHz_highsnr, correctedflux,'k-', linewidth=2.0)
plt.plot(freqMHz_highsnr, scatflux,'r--', linewidth=2.0)
plt.fill_between(freqMHz_highsnr,scatflux,correctedflux, alpha=alfval,facecolor='r')
eb = plt.errorbar(freqMHz_highsnr,correctedflux,yerr=sigmaFlux, fmt=markr,markersize=10.0, alpha=1.0,capthick=2,linewidth=1.5)
eb[-1][0].set_linestyle(ls)
#plt.errorbar(freqMHz_highsnr, unscatflux,yerr=sigmaFlux, fmt=markr,markersize=10.0, alpha=alfval)
plt.title('Flux Spectrum', fontsize=12)
plt.xticks(fontsize=12)
plt.yticks(fontsize=12)
plt.xlabel(r'$\nu$ (MHz)',fontsize=12)
plt.ylabel(r'Calibrated flux (mJy)',fontsize=12)
return freqMHz_highsnr, taussec_highsnr, lmfitstdssec_highsnr
from lmfit.models import ExponentialModel, GaussianModel
N = 80
# dat = np.loadtxt('Gauss2.dat')
x = np.linspace(0, 40, N, endpoint=False)
y1 = np.exp(-x)
y2 = np.exp(-5 * x)
y = y1 + y2
# plt.plot(x, y1, 'b')
# plt.plot(x, y2, 'k--')
# plt.plot(x, y , 'r-')
# plt.show()
exp1 = ExponentialModel(prefix='e1_')
exp2 = ExponentialModel(prefix='e2_')
pars1 = exp1.guess(y, x=x)
pars2 = exp2.guess(y, x=x)
# pars1 = exp_mod.guess(y[:ix1], x=x[:ix1])
# pars2 = gauss1.guess(y[ix1:ix2], x=x[ix1:ix2])
# pars3 = gauss2.guess(y[ix2:ix3], x=x[ix2:ix3])
pars = pars1 + pars2
mod = exp1 + exp2
out = mod.fit(y, pars, x=x)
print(out.fit_report(min_correl=0.5))
plt.plot(x, y, 'b')
plt.plot(x, out.init_fit, 'k--')
plt.plot(x, out.best_fit, 'r-')
class gene_set():
def __init__(self,gene_id, cluster):
self.cluster= cluster
self.gene_id = gene_id
self.norm_vals = [float(y) for y in [x[1:] for x in reference if x[0] == gene_id][0]] #TPM.
self.get_model()
#self.half_life()
#self.printing()
#self.saving()
def get_model(self):
self.x = np.array([0,1,2,6,12,24])
self.y = np.array(self.norm_vals)
self.model_flag = None
self.model_list = []
# First model with Gaussian curve.
self.background1 = ExponentialModel(prefix='e1_')
self.pars1 = self.background1.guess(self.y, x=self.x)
self.peak1 = GaussianModel(prefix='p1_')
self.pars1 += self.peak1.guess(self.y, x=self.x)
self.comp_mod1 = self.peak1 + self.background1
self.init1 = self.comp_mod1.eval(self.pars1, x=self.x)
self.comp_out1 = self.comp_mod1.fit(self.y, x=self.x, fit_kws={'nan_policy': 'omit'})
self.comp_list1 = self.comp_out1.fit_report().split('\n')
self.comp_chisq1 = float(self.comp_list1[6][-5:])
# Second model with Voigt curve.
self.background2 = ExponentialModel(prefix='e2_')
self.pars2 = self.background2.guess(self.y, x=self.x)
self.peak2 = VoigtModel(prefix='p2_')
self.pars2 += self.peak2.guess(self.y, x=self.x)
self.comp_mod2 = self.peak2 + self.background2
self.init2 = self.comp_mod2.eval(self.pars2, x=self.x)
self.comp_out2 = self.comp_mod2.fit(self.y, x=self.x, fit_kws={'nan_policy': 'omit'})
self.comp_list2 = self.comp_out2.fit_report().split('\n')
self.comp_chisq2 = float(self.comp_list2[6][-5:])
# Exponential model for reference
self.exp_mod = ExponentialModel(prefix='onlye_')
self.pars = self.exp_mod.guess(self.y, x=self.x)
self.init = self.exp_mod.eval(self.pars, x=self.x)
self.exp_out = self.exp_mod.fit(self.y, x=self.x, missing='drop')
self.exp_list = self.exp_out.fit_report().split('\n')
self.exp_chisq = float(self.exp_list[6][-5:])
self.model_list = [self.comp_chisq1, self.comp_chisq2, self.exp_chisq]
if np.count_nonzero(np.isinf(self.comp_out1.best_fit)) == 5 and np.count_nonzero(np.isinf(self.comp_out2.best_fit)):
model_flag = "exponential"
self.out = self.exp_out
elif len(self.model_list) == len(set(self.model_list)):
if min(self.model_list) == self.comp_chisq1:
self.model_flag = "Gaussian compound"
self.out = self.comp_out1
elif min(self.model_list) == self.comp_chisq2:
self.model_flag = "Voigt compound"
self.out = self.comp_out2
elif min(self.model_list) == self.exp_chisq:
self.model_flag = "exponential"
self.out = self.exp_out
elif len(self.model_list) != len(set(self.model_list)):
if min(self.model_list) == self.comp_chisq1:
self.model_flag = "Gaussian compound"
self.out = self.comp_out1
elif min(self.model_list) == self.comp_chisq2:
self.model_flag = "Voigt compound"
self.out = self.comp_out2
elif min(self.model_list) == self.exp_chisq:
self.model_flag = "exponential"
self.out = self.exp_out
if min(self.model_list) == self.comp_chisq1 and self.comp_chisq1 == self.comp_chisq2:
self.model_flag = "Both compounds"
self.out = self.comp_out2
if min(self.model_list) == self.comp_chisq2 and self.comp_chisq2 == self.exp_chisq:
self.model_flag = "Voigt compound and exponential"
self.out = self.comp_out2
if min(self.model_list) == self.exp_chisq and self.exp_chisq == self.comp_chisq1:
self.model_flag = "Gaussian compound and exponential"
self.out = self.comp_out1
return self.comp_out1, self.comp_chisq1, self.comp_out2, self.comp_chisq2, self.exp_out, self.exp_chisq, self.model_flag
y = dat[:, 0]
exp_mod = ExponentialModel(prefix='exp_')
gauss1 = GaussianModel(prefix='g1_')
gauss2 = GaussianModel(prefix='g2_')
def index_of(arrval, value):
"return index of array *at or below* value "
if value < min(arrval): return 0
return max(np.where(arrval<=value)[0])
ix1 = index_of(x, 75)
ix2 = index_of(x, 135)
ix3 = index_of(x, 175)
pars1 = exp_mod.guess(y[:ix1], x=x[:ix1])
pars2 = gauss1.guess(y[ix1:ix2], x=x[ix1:ix2])
pars3 = gauss2.guess(y[ix2:ix3], x=x[ix2:ix3])
pars = pars1 + pars2 + pars3
mod = gauss1 + gauss2 + exp_mod
out = mod.fit(y, pars, x=x)
print(out.fit_report(min_correl=0.5))
plt.plot(x, y)
plt.plot(x, out.init_fit, 'k--')
plt.plot(x, out.best_fit, 'r-')
plt.show()
#<end examples/doc_nistgauss2.py>
def calculateQ_2peaks(filename, time, taper, lambda0, slope,
modulation_coefficient, rg):
q = readlvm(filename)
q = q.ravel()
q = q / taper - 1
valley = q.min()
valley_index = np.argmin(q)
max_height = -1 * q[valley_index]
q_valley = q[valley_index - rg:valley_index + rg]
l_valley = time[valley_index - rg:valley_index + rg]
q_valley = q_valley * -1
peaks, peak_info = find_peaks(q_valley,
height=max_height * 0.6,
prominence=0.05,
distance=50)
#results_half = peak_widths(q_valley, peaks, rel_height=0.5)
if len(peaks) != 2:
print("Wrong peaks with num:", len(peaks))
return None, None, None, None, None, None
x = np.asarray(list(range(0, 2 * rg)))
y = q_valley
# One peak guess to get width
g_mod = LorentzianModel()
g_pars = g_mod.guess(y, x=x)
g_out = g_mod.fit(y, g_pars, x=x)
g_res = g_out.fit_report()
g_info = g_res.split("\n")
g_variables = parse_info(g_info, 1)
guessedWidth = float(g_variables['fwhm']) / 2
exp_mod = ExponentialModel(prefix='exp_')
pars = exp_mod.guess(y, x=x)
lorenz1 = LorentzianModel(prefix='l1_')
pars.update(lorenz1.make_params())
pars['l1_center'].set(peaks[0])
pars['l1_sigma'].set(guessedWidth)
pars['l1_amplitude'].set(np.pi * guessedWidth * q_valley[peaks[0]])
lorenz2 = LorentzianModel(prefix='l2_')
pars.update(lorenz2.make_params())
pars['l2_center'].set(peaks[1])
pars['l2_sigma'].set(guessedWidth)
pars['l2_amplitude'].set(np.pi * guessedWidth * q_valley[peaks[1]])
mod = lorenz1 + lorenz2 + exp_mod
init = mod.eval(pars, x=x)
out = mod.fit(y, pars, x=x)
res = out.fit_report()
info = res.split("\n")
variables = parse_info(info, 2)
l1_h_res, l1_w_res, l1_c_res = variables['l1_height'], variables[
'l1_fwhm'], variables['l1_center']
print(l1_h_res, l1_w_res, l1_c_res)
l1 = lambda0 + l_valley[int(
float(l1_c_res))] * slope * modulation_coefficient
d_lambda1 = (l_valley[1] - l_valley[0]
) * float(l1_w_res) * slope * modulation_coefficient
Q1 = l1 / d_lambda1
l2_h_res, l2_w_res, l2_c_res = variables['l2_height'], variables[
'l2_fwhm'], variables['l2_center']
print(l2_h_res, l2_w_res, l2_c_res)
l2 = lambda0 + l_valley[int(
float(l2_c_res))] * slope * modulation_coefficient
d_lambda2 = (l_valley[1] - l_valley[0]
) * float(l2_w_res) * slope * modulation_coefficient
Q2 = l2 / d_lambda2
return Q1, float(l1_h_res) * 100, l1, Q2, float(l2_h_res) * 100, l2
print("***** Iteration ",i," *****")
data = langevin.time_series(A=A, D=D, delta_t=delta_t, N=N)
data_results = [data[0]**2, data[-1]**2, np.sum(data[1:-2]**2), np.sum(data[:-1]*data[1:])]
# calculate autocorrelation function
f = np.fft.rfft(data)
acf = np.fft.irfft(f * np.conjugate(f))
acf = np.fft.fftshift(acf) / N
autocorr = acf[int(N / 2):]
y = autocorr[:min(int(N / 2), P)]
t = np.arange(min(int(N / 2), P))
mod = ExponentialModel()
pars = mod.guess(y, x=t)
try:
out = mod.fit(y, pars, x=t)
except:
fit_results = np.zeros(4)
print('fit did not work')
else:
fit_results = np.array([out.values['decay']*delta_t,
np.sqrt(out.covar[0,0])*delta_t,
out.values['amplitude'],
np.sqrt(out.covar[1,1])])
print(out.fit_report(min_correl=0.25))
trace = sm.run(x=data,
aB=alpha_B,
bB=beta_B,
amps.append(np.array(amp))
amps_std.append(np.array(amp_err))
print(o_i.fit_report(min_correl=0.5))
amps = np.vstack(amps)
amps_std = np.vstack(amps_std)
#print(out.fit_report(min_correl=0.5))
##ax.plot_wireframe(X, Y, Z_noise.reshape(50,50),color="r")
#ax.plot_wireframe(X, Y, Z_conv.reshape(50,50),color="r")
ax.plot_wireframe(X, Y, test_data_noise.reshape(50,50),color="r")
colors = ["r","g","b","orange","k","c"]
#for i,c in zip(exp_test_data,colors):
# ax.plot_wireframe(X, Y, i.reshape(50,50),color=c)
ax.plot_wireframe(X,Y,out.best_fit.reshape(50,50),color="b")
##ax.contour(X,Y,Z)
ax.set_ylabel("y")
ax.set_xlabel("x")
plt.show()
t = np.arange(6)
for i in range(len(exp_test_data)):
mod = ExponentialModel()
pars = mod.guess(amps[:,i],x=t)
fit = mod.fit(amps[:,i],pars,x=t)
plt.errorbar(t,amps[:,i],yerr=amps_std[:,i],fmt="ro")
plt.plot(t,fit.best_fit,"k--")
plt.show()
amps.append(np.array(amp))
amps_std.append(np.array(amp_err))
print(o_i.fit_report(min_correl=0.5))
amps = np.vstack(amps)
amps_std = np.vstack(amps_std)
#print(out.fit_report(min_correl=0.5))
##ax.plot_wireframe(X, Y, Z_noise.reshape(50,50),color="r")
#ax.plot_wireframe(X, Y, Z_conv.reshape(50,50),color="r")
ax.plot_wireframe(X, Y, test_data_noise.reshape(50, 50), color="r")
colors = ["r", "g", "b", "orange", "k", "c"]
#for i,c in zip(exp_test_data,colors):
# ax.plot_wireframe(X, Y, i.reshape(50,50),color=c)
ax.plot_wireframe(X, Y, out.best_fit.reshape(50, 50), color="b")
##ax.contour(X,Y,Z)
ax.set_ylabel("y")
ax.set_xlabel("x")
plt.show()
t = np.arange(6)
for i in range(len(exp_test_data)):
mod = ExponentialModel()
pars = mod.guess(amps[:, i], x=t)
fit = mod.fit(amps[:, i], pars, x=t)
plt.errorbar(t, amps[:, i], yerr=amps_std[:, i], fmt="ro")
plt.plot(t, fit.best_fit, "k--")
plt.show()
def calculate_feature2(image_path, value=[1,0,0,0,1,0], return_image=False):
debug_image = []
image = cv2.imread(image_path)
from detector_borde_plastico import detector_borde_plastico
image_bp, images_dp_debug = detector_borde_plastico(image)
debug_image.extend(images_dp_debug)
image_bottom = image_bp[int(0.7*image_bp.shape[0]):image_bp.shape[0],:,:]
#image_sharr = cv2.Scharr(image_bottom, cv2.CV_32F, 2, 0)
image_bottom_lab = cv2.cvtColor(image_bottom, cv2.COLOR_BGR2LAB)
#features = image_bottom_lab.flatten()[:500]
#debug_image.append(image_bottom_lab[:,:,0])
#debug_image.append(image_bottom_lab[:,:,2])
image_lab_a = image_bottom_lab[:, :, 1]
image_lab_b = image_bottom_lab[:, :, 2]
debug_image.append(image_lab_a)
debug_image.append(image_lab_b)
thresh = cv2.threshold(image_lab_a, 0, 255,
cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]
#debug_image.append(thresh)
_, contours, hierarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
# bigest_area = amg
max_area = 0
for cnt in contours:
area = cv2.contourArea(cnt)
if max_area < area:
max_area = area
biggest_area = cnt
"""
# points = [(100,h1[1][1]-h1[0][1]-5), (200, h1[1][1]-h1[0][1]-5), (300, h1[1][1]-h1[0][1]-5)]
bad = 0
good = 0
for x in xrange(0, th2.shape[1], 10):
p = (x, th2.shape[0] - 5)
if cv2.pointPolygonTest(biggest_area, p, False) <= 0:
bad += 1
else:
good += 1
# print "good:", good, "bad:", bad
isgood = False
if bad < 25:
isgood = True
"""
mask_pasto = np.zeros(thresh.shape, np.uint8)
cv2.drawContours(mask_pasto, [biggest_area], 0, 255, 3)
debug_image.append(mask_pasto)
#features = [mask_pasto == 1]
"""
image_kmeans = kmeans(image_bottom_lab, K=40)
image_kmeans_rgb = cv2.cvtColor(image_kmeans, cv2.COLOR_LAB2BGR)
debug_image.append(image_kmeans_rgb)
image_kmeans_rgb_cut = image_kmeans_rgb[0:image_kmeans_rgb.shape[0], int(image_kmeans_rgb.shape[1]*0.167):int(image_kmeans_rgb.shape[1]*0.905)]
debug_image.append(image_kmeans_rgb_cut)
image_kmeans2 = kmeans(image_kmeans_rgb_cut, K=5)
#image_kmeans2_rgb = cv2.cvtColor(image_kmeans2, cv2.COLOR_LAB2BGR)
debug_image.append(image_kmeans2)
lowerBound = np.array(value[0:3])*255.0
upperBound = np.array(value[3:6])*255.0
# this gives you the mask for those in the ranges you specified,
# but you want the inverse, so we'll add bitwise_not...
image_inrange = cv2.inRange(image_kmeans2, lowerBound, upperBound)
image_inrange_color = cv2.bitwise_not(image_kmeans_rgb_cut, mask=image_inrange)
debug_image.append(image_inrange_color)
"""
"""
cv2.kmeans(image_bottom_lab, 40, )
from sklearn.cluster import MiniBatchKMeans
image_bottom_lab_reshape = image_bottom_lab.reshape((image_bottom_lab.shape[0] * image_bottom_lab.shape[1], 3))
clt = MiniBatchKMeans(n_clusters=10)
labels = clt.fit_predict(image_bottom_lab_reshape)
quant = clt.cluster_centers_.astype("uint8")[labels]
quant = quant.reshape(image_bottom_lab.shape)
image_bottom_lab_res = image_bottom_lab_reshape.reshape(image_bottom_lab.shape)
# convert from L*a*b* to RGB
quant = cv2.cvtColor(quant, cv2.COLOR_LAB2BGR)
image_bottom_lab_res = cv2.cvtColor(image_bottom_lab_res, cv2.COLOR_LAB2BGR)
sobelx64f = cv2.Sobel(image_bottom, cv2.CV_64F, 1, 0, ksize=3)
abs_sobel64f = np.absolute(sobelx64f)
sobel_8u = np.uint8(abs_sobel64f)
lowerBound = np.array(value[0:3])*255.0
upperBound = np.array(value[3:6])*255.0
# this gives you the mask for those in the ranges you specified,
# but you want the inverse, so we'll add bitwise_not...
image_inrange = cv2.inRange(sobel_8u, lowerBound, upperBound)
image_inrange_color = cv2.bitwise_not(image_bottom, mask=image_inrange)
features = image.flatten()[:500]
debug_image.append(image_bottom)
debug_image.append(sobel_8u)
debug_image.append(image_inrange)
debug_image.append(image_inrange_color)
debug_image.append(quant)
debug_image.append(image_bottom_lab_res)
"""
x = []
y = []
for x_ in xrange(mask_pasto.shape[1]):
for y_ in xrange(10, mask_pasto.shape[0]):
if mask_pasto[y_,x_] == 255:
x.append(x_)
y.append(y_)
break
x = np.array(x)
y = np.array(y)
x1 = x[x<(mask_pasto.shape[1]/2)]
y1 = y[x<(mask_pasto.shape[1]/2)]
x1 = x1[y1>(mask_pasto.shape[0]/2)]
y1 = y1[y1>(mask_pasto.shape[0]/2)]
y1 = y1.max() - y1
x = x-x[0] # x offset
mask_pasto2 = np.zeros(thresh.shape, np.uint8)
for i in xrange(len(x1)):
mask_pasto2[y1[i], x1[i]] = 255
debug_image.append(mask_pasto2)
x2 = x[x>(mask_pasto.shape[1]/2)]
y2 = y[x>(mask_pasto.shape[1]/2)]
y2 = y2 - y2.max()
from lmfit.models import ExponentialModel
exp_mod = ExponentialModel(prefix='exp_')
pars = exp_mod.guess(y1, x=x1)
out = exp_mod.fit(y, pars, x=x)
print "error en amplitud:", out.params['exp_amplitude'].stderr,
print "error en decay:", out.params['exp_decay'].stderr,
features = np.array([out.best_values['exp_decay'], out.params['exp_decay'].stderr])
#print(out.fit_report(min_correl=0.5))
#print pars
if return_image:
return features, debug_image
return features
class gene_set():
def __init__(self, gene_id, cluster):
self.cluster = cluster
self.gene_id = gene_id
self.norm_vals = [
float(y) for y in [x[1:] for x in reference if x[0] == gene_id][0]
] #TPM.
self.get_model()
self.def_peaks()
self.model_resetting()
self.half_life()
#self.printing()
self.saving()
def get_model(self):
self.x = np.array([0, 1, 2, 6, 12, 24])
self.y = np.array(self.norm_vals)
# Compound model with Voigt curve.
self.background = ExponentialModel(prefix='b_')
self.pars = self.background.guess(self.y, x=self.x)
self.peak = VoigtModel(prefix='p_')
self.pars += self.peak.guess(self.y, x=self.x)
self.comp_mod = self.peak + self.background
self.init = self.comp_mod.eval(self.pars, x=self.x)
self.comp_out = self.comp_mod.fit(
self.y, x=self.x,
fit_kws={'nan_policy': 'propagate'
}) # instead of 'omit', it keeps up the zero vals.
self.comp_list = self.comp_out.fit_report().split('\n')
self.comp_chisq = float(self.comp_list[6][-5:])
self.out = self.comp_out
self.chisq = float(self.comp_list[6][-5:])
self.usedmod = self.comp_mod
self.model_flag = "composite (exponential+Voigt)"
return self.comp_out, self.comp_chisq, self.out, self.chisq, self.usedmod, self.model_flag
def def_peaks(self):
self.bestfit = self.comp_out.best_fit
self.idx = np.argmin(np.abs(self.bestfit - 0.5))
self.mysort = np.argsort(np.abs(self.bestfit - 0.5))
self.peak_flag = None
if all(i > 0.5 for i in self.bestfit):
# Meaning that it is never reaching the half-life, and we don't do extrapolation (not enough data points).
self.min = 0
self.max = 0
self.peak_flag = "No predictable half-life"
else:
if self.bestfit[self.idx] == 0.5:
# If by accident one time point hits the half-life.
self.half_life_y = self.bestfit[self.idx]
self.half_life_x = self.idx
self.peak_flag = "Exact compound half-life"
elif self.bestfit[0] > 0.5 and self.bestfit[1] < 0.5:
self.min = self.x[0]
self.max = self.x[1]
self.peak_flag = "Compound"
elif self.idx == 5 and self.bestfit[self.idx - 1] < 0.5:
# Last value crosses only
self.max = self.x[self.idx]
self.min = self.x[self.idx - 1]
self.peak_flag = "Compound"
elif np.abs(self.idx - self.mysort[1]) == 1:
if self.bestfit[self.idx] < 0.5:
self.min = self.x[self.idx - 1]
self.max = self.x[self.idx]
self.peak_flag = "Compound"
elif self.bestfit[self.idx] > 0.5:
self.min = self.x[self.idx]
self.max = self.x[self.idx + 1]
self.peak_flag = "Compound"
elif np.abs(self.idx - self.mysort[1]) > 1:
# Meaning that the steps are not linear, there's a bump.
if self.bestfit[self.idx] < 0.5:
self.min = self.x[self.idx - 1]
self.max = self.x[self.idx]
self.peak_flag = "Compound"
elif self.bestfit[self.idx] > 0.5 and self.bestfit[
self.mysort[1]] < 0.5:
if self.bestfit[self.idx + 1] < 0.5:
self.min = self.x[self.idx]
self.max = self.x[self.idx + 1]
self.peak_flag = "Compound"
#resetting!!
else:
self.min = self.x[self.mysort[1] - 1]
self.max = self.x[self.mysort[1]]
self.peak_flag = "Resetting"
elif self.bestfit[self.idx] > 0.5 and self.bestfit[
self.mysort[1]] > 0.5:
if self.bestfit[self.idx + 1] < 0.5:
self.min = self.x[self.idx]
self.max = self.x[self.idx + 1]
self.peak_flag = "Compound"
#resetting!!
elif self.bestfit[self.idx + 1] > 0.5 and self.bestfit[
self.mysort[1] + 1] < 0.5:
self.min = self.x[self.mysort[1] - 1]
self.max = self.x[self.mysort[1]]
self.peak_flag = "Resetting"
return self.min, self.max, self.peak_flag, self.bestfit
def model_resetting(self):
if self.peak_flag != "Resetting":
#go for the previous method
pass
elif self.peak_flag == "Resetting":
# mostly for plotting, half-life needs new zeros
self.scnd_peak = np.sort(self.bestfit)[-2]
self.scnd_idx = np.argsort(self.bestfit)[-2]
self.newzero = self.x[self.scnd_idx]
# Cutting the new time scale, reset to 0.
self.x2 = np.array(
[i - self.newzero for i in self.x[self.scnd_idx:]])
#x2 = np.array([i for i in x[scnd_idx:]])
# Re-normalized and cutted array
self.y2 = np.array(
[i / self.y[self.scnd_idx] for i in self.y[self.scnd_idx:]])
#newarray = myarray[scnd_idx:]
self.exp_mod = ExponentialModel(prefix='e_')
self.pars = self.exp_mod.guess(self.y2, x=self.x2)
self.init = self.exp_mod.eval(self.pars, x=self.x2)
self.exp_out = self.exp_mod.fit(self.y2, x=self.x2, missing='drop')
self.exp_list = self.exp_out.fit_report().split('\n')
self.exp_chisq = float(self.exp_list[6][-5:])
self.out = self.exp_out
self.chisq = float(self.exp_list[6][-5:])
self.usedmod = self.exp_mod
self.bestfit = self.exp_out.best_fit
self.idx = np.argmin(np.abs(self.bestfit - 0.5))
self.mysort = np.argsort(np.abs(self.bestfit - 0.5))
self.peak_flag = None
if self.bestfit[self.idx] < 0.5:
self.min = self.x2[self.idx - 1]
self.max = self.x2[self.idx]
self.peak_flag = "Resetted exponential"
elif self.bestfit[self.idx] > 0.5:
self.min = self.x2[self.idx]
self.max = self.x2[self.idx + 1]
self.peak_flag = "Resetted exponential"
# For printing.
if len(self.bestfit) < 6:
l = [self.bestfit[-1]] * (6 - len(self.bestfit))
self.bestfit = np.append(self.bestfit, l)
self.x = self.x2
self.y = self.y2
self.model_flag = "exponential"
return self.min, self.max, self.peak_flag, self.bestfit, self.x, self.out, self.chisq, self.usedmod, self.model_flag
def half_life(self):
self.new_x = np.array([0])
self.hl_eval = np.array([0])
self.hl_array = np.array([0])
self.hl_coord = np.array([0])
self.step = None
if self.max == 0:
self.half_life_y = 0
self.half_life_x = 0
self.peak_flag = "No predictable half-life"
else:
self.half_life_y = 0
self.half_life_x = 0
self.step = 0.1
self.max_allowed = 3
self.attempt = 0
#while self.attempt < 3 or self.half_life_y == 0:
while self.half_life_y == 0 and self.attempt < 3:
self.attempt += 1
self.step = self.step / 100
self.ranging = np.arange(
self.min, self.max, self.step
) # normally it 0.001, but the slope is so radical, can't catxh half-life.
for j in np.nditer(self.ranging):
self.new_x = np.array([j])
#self.h = self.out.eval_components(self.out.params,x=self.new_x)
#self.hl_eval = list(self.h.values())[-1]
self.hl_eval = self.out.eval(self.out.params, x=self.new_x)
if self.hl_eval >= 0.50 and self.hl_eval <= 0.51:
self.hl_array = np.append(self.hl_array, self.hl_eval)
self.hl_coord = np.append(self.hl_coord, self.new_x)
self.half_life_id = np.argmin(np.abs(self.hl_array - 0.5))
self.half_life_y = self.hl_array[self.half_life_id]
self.half_life_x = self.hl_coord[self.half_life_id]
self.peak_flag = self.peak_flag
if self.half_life_y == 0:
self.peak_flag = "Above permitted interpolation iterations"
return self.half_life_y, self.half_life_x, self.peak_flag
def saving(self):
with open('model_fit_c5_average_filtering_compound.txt', 'a') as f:
f.write(
"%s\t%s\t%s\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%s\n"
% (self.gene_id, self.cluster, self.model_flag, self.chisq,
self.half_life_y, self.half_life_x, self.norm_vals[0],
self.norm_vals[1], self.norm_vals[2], self.norm_vals[3],
self.norm_vals[4], self.norm_vals[5], self.bestfit[0],
self.bestfit[1], self.bestfit[2], self.bestfit[3],
self.bestfit[4], self.bestfit[5], self.peak_flag))
def calculate_feature(image_path,
value=[1, 0, 0, 0, 1, 0],
return_image=False):
debug_image = []
image = cv2.imread(image_path)
from detector_borde_plastico import detector_borde_plastico
image_bp, images_dp_debug = detector_borde_plastico(image)
debug_image.extend(images_dp_debug)
#features = [mask_pasto == 1]
"""
image_kmeans = kmeans(image_bottom_lab, K=40)
image_kmeans_rgb = cv2.cvtColor(image_kmeans, cv2.COLOR_LAB2BGR)
debug_image.append(image_kmeans_rgb)
image_kmeans_rgb_cut = image_kmeans_rgb[0:image_kmeans_rgb.shape[0], int(image_kmeans_rgb.shape[1]*0.167):int(image_kmeans_rgb.shape[1]*0.905)]
debug_image.append(image_kmeans_rgb_cut)
image_kmeans2 = kmeans(image_kmeans_rgb_cut, K=5)
#image_kmeans2_rgb = cv2.cvtColor(image_kmeans2, cv2.COLOR_LAB2BGR)
debug_image.append(image_kmeans2)
lowerBound = np.array(value[0:3])*255.0
upperBound = np.array(value[3:6])*255.0
# this gives you the mask for those in the ranges you specified,
# but you want the inverse, so we'll add bitwise_not...
image_inrange = cv2.inRange(image_kmeans2, lowerBound, upperBound)
image_inrange_color = cv2.bitwise_not(image_kmeans_rgb_cut, mask=image_inrange)
debug_image.append(image_inrange_color)
"""
"""
cv2.kmeans(image_bottom_lab, 40, )
from sklearn.cluster import MiniBatchKMeans
image_bottom_lab_reshape = image_bottom_lab.reshape((image_bottom_lab.shape[0] * image_bottom_lab.shape[1], 3))
clt = MiniBatchKMeans(n_clusters=10)
labels = clt.fit_predict(image_bottom_lab_reshape)
quant = clt.cluster_centers_.astype("uint8")[labels]
quant = quant.reshape(image_bottom_lab.shape)
image_bottom_lab_res = image_bottom_lab_reshape.reshape(image_bottom_lab.shape)
# convert from L*a*b* to RGB
quant = cv2.cvtColor(quant, cv2.COLOR_LAB2BGR)
image_bottom_lab_res = cv2.cvtColor(image_bottom_lab_res, cv2.COLOR_LAB2BGR)
sobelx64f = cv2.Sobel(image_bottom, cv2.CV_64F, 1, 0, ksize=3)
abs_sobel64f = np.absolute(sobelx64f)
sobel_8u = np.uint8(abs_sobel64f)
lowerBound = np.array(value[0:3])*255.0
upperBound = np.array(value[3:6])*255.0
# this gives you the mask for those in the ranges you specified,
# but you want the inverse, so we'll add bitwise_not...
image_inrange = cv2.inRange(sobel_8u, lowerBound, upperBound)
image_inrange_color = cv2.bitwise_not(image_bottom, mask=image_inrange)
features = image.flatten()[:500]
debug_image.append(image_bottom)
debug_image.append(sobel_8u)
debug_image.append(image_inrange)
debug_image.append(image_inrange_color)
debug_image.append(quant)
debug_image.append(image_bottom_lab_res)
"""
x = []
y = []
for x_ in xrange(mask_pasto.shape[1]):
for y_ in xrange(10, mask_pasto.shape[0]):
if mask_pasto[y_, x_] == 255:
x.append(x_)
y.append(y_)
break
x = np.array(x)
y = np.array(y)
x1 = x[x < (mask_pasto.shape[1] / 2)]
y1 = y[x < (mask_pasto.shape[1] / 2)]
x1 = x1[y1 > (mask_pasto.shape[0] / 2)]
y1 = y1[y1 > (mask_pasto.shape[0] / 2)]
y1 = y1.max() - y1
x = x - x[0] # x offset
mask_pasto2 = np.zeros(thresh.shape, np.uint8)
for i in xrange(len(x1)):
mask_pasto2[y1[i], x1[i]] = 255
debug_image.append(mask_pasto2)
x2 = x[x > (mask_pasto.shape[1] / 2)]
y2 = y[x > (mask_pasto.shape[1] / 2)]
y2 = y2 - y2.max()
from lmfit.models import ExponentialModel
exp_mod = ExponentialModel(prefix='exp_')
pars = exp_mod.guess(y1, x=x1)
out = exp_mod.fit(y, pars, x=x)
print "error en amplitud:", out.params['exp_amplitude'].stderr,
print "error en decay:", out.params['exp_decay'].stderr,
features = np.array(
[out.best_values['exp_decay'], out.params['exp_decay'].stderr])
#print(out.fit_report(min_correl=0.5))
#print pars
if return_image:
return features, debug_image
return features
#!/usr/bin/env python
# <examples/doc_builtinmodels_nistgauss.py>
import matplotlib.pyplot as plt
import numpy as np
from lmfit.models import ExponentialModel, GaussianModel
dat = np.loadtxt('NIST_Gauss2.dat')
x = dat[:, 1]
y = dat[:, 0]
exp_mod = ExponentialModel(prefix='exp_')
pars = exp_mod.guess(y, x=x)
gauss1 = GaussianModel(prefix='g1_')
pars.update(gauss1.make_params())
pars['g1_center'].set(105, min=75, max=125)
pars['g1_sigma'].set(15, min=3)
pars['g1_amplitude'].set(2000, min=10)
gauss2 = GaussianModel(prefix='g2_')
pars.update(gauss2.make_params())
pars['g2_center'].set(155, min=125, max=175)
pars['g2_sigma'].set(15, min=3)
pars['g2_amplitude'].set(2000, min=10)
mod = gauss1 + gauss2 + exp_mod
class gene_set():
def __init__(self, gene_id, cluster):
self.cluster = cluster
self.gene_id = gene_id
self.norm_vals = [
float(y) for y in [x[1:] for x in reference if x[0] == gene_id][0]
] #TPM.
self.get_model()
self.half_life()
self.printing()
self.saving()
def get_model(self):
self.x = np.array([0, 1, 2, 6, 12, 24])
self.y = np.array(self.norm_vals)
#x = np.array([0,1,2,6,12,24])
# Exponential model for reference
self.exp_mod = ExponentialModel(prefix='onlye_')
self.pars = self.exp_mod.guess(self.y, x=self.x)
self.init = self.exp_mod.eval(self.pars, x=self.x)
self.exp_out = self.exp_mod.fit(self.y, x=self.x, missing='drop')
self.exp_list = self.exp_out.fit_report().split('\n')
self.exp_chisq = float(self.exp_list[6][-5:])
return self.exp_out, self.exp_chisq
def half_life(self):
self.new_x = np.array([0])
self.hl_eval = np.array([0])
self.hl_array = np.array([0])
self.hl_coord = np.array([0])
self.bestfit = self.exp_out.best_fit
self.idx = np.argmin(np.abs(self.bestfit - 0.5))
if self.idx == 5 and self.bestfit[self.idx - 1] < 0.5:
self.bestfit = self.exp_out.best_fit[:-1]
self.idx = np.argmin(np.abs(self.bestfit - 0.5))
if self.bestfit[self.idx] == 0.5:
self.half_life_y = self.bestfit[self.idx]
self.half_life_x = self.idx
elif 0.5 > self.bestfit[self.idx] and self.bestfit[self.idx - 1] > 0.5:
self.max = self.x[self.idx]
self.min = self.x[self.idx - 1]
elif 0.5 < self.bestfit[self.idx] and self.bestfit[
self.idx] == self.bestfit[5]:
self.min = 0
self.max = 0
elif 0.5 < self.bestfit[self.idx] and self.bestfit[self.idx + 1] < 0.5:
self.min = self.x[self.idx]
self.max = self.x[self.idx + 1]
elif 0.5 < self.bestfit[self.idx] and self.bestfit[
self.idx + 1] > 0.5 and self.bestfit[self.idx + 2] < 0.5:
self.min = self.x[self.idx + 1]
self.max = self.x[self.idx + 2]
elif 0.5 > self.bestfit[self.idx] and self.bestfit[
self.idx + 1] < 0.5 and self.bestfit[self.idx - 2] > 0.5:
self.min = self.x[self.idx - 2]
self.max = self.x[self.idx]
self.ranging = np.arange(self.min, self.max, 0.001)
if self.max > 0:
# if self.min > 0 and self.max > 0:
for j in np.nditer(self.ranging):
self.new_x = np.array([j])
self.hl_eval = self.exp_out.eval(self.exp_out.params,
x=self.new_x)
if self.hl_eval >= 0.50 and self.hl_eval <= 0.51:
self.hl_array = np.append(self.hl_array, self.hl_eval)
self.hl_coord = np.append(self.hl_coord, self.new_x)
self.half_life_id = np.argmin(np.abs(self.hl_array - 0.5))
self.half_life_y = self.hl_array[self.half_life_id]
self.half_life_x = self.hl_coord[self.half_life_id]
self.bestfit = self.exp_out.best_fit
else:
self.half_life_y = 0
self.half_life_x = 0
self.bestfit = self.exp_out.best_fit
return self.half_life_y, self.half_life_x
def printing(self):
print(self.gene_id, self.cluster, "exponential", self.exp_chisq,
self.half_life_y, self.half_life_x, self.norm_vals[0],
self.norm_vals[1], self.norm_vals[2], self.norm_vals[3],
self.norm_vals[4], self.norm_vals[5], self.bestfit[0],
self.bestfit[1], self.bestfit[2], self.bestfit[3],
self.bestfit[4], self.bestfit[5])
def saving(self):
with open('model_fit_c5_average_filtering_newexp.txt', 'a') as f:
(f.write(
"%s\t%s\t%s\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\n"
% (self.gene_id, self.cluster, "exponential", self.exp_chisq,
self.half_life_y, self.half_life_x, self.norm_vals[0],
self.norm_vals[1], self.norm_vals[2], self.norm_vals[3],
self.norm_vals[4], self.norm_vals[5], self.bestfit[0],
self.bestfit[1], self.bestfit[2], self.bestfit[3],
self.bestfit[4], self.bestfit[5])))
result_array = None
for i in range(M):
print("***** Iteration ", i, " *****")
data = langevin.time_series(A=A, D=D, delta_t=delta_t, N=N)
# calculate autocorrelation function
f = np.fft.rfft(data)
acf = np.fft.irfft(f * np.conjugate(f))
acf = np.fft.fftshift(acf) / N
autocorr = acf[int(N / 2):]
y = autocorr[:min(int(N / 2), P)]
t = np.arange(min(int(N / 2), P))
mod = ExponentialModel()
pars = mod.guess(y, x=t)
try:
out = mod.fit(y, pars, x=t)
except:
fit_results = np.zeros(4)
print('fit did not work')
else:
fit_results = np.array([
out.values['decay'] * delta_t,
np.sqrt(out.covar[0, 0]) * delta_t, out.values['amplitude'],
np.sqrt(out.covar[1, 1])
])
print(out.fit_report(min_correl=0.25))
trace = sm.run(x=data,
aD=alpha_D,
data = pd.read_csv("fits.csv")
groups = data.groupby("assignment")
for ind, group in groups:
plt.figure(figsize=(4, 3))
plt.errorbar(
group.vclist,
group.amp,
yerr=group.amp_err,
fmt="o",
markersize=10,
label=group.assignment.iloc[0],
c="#33a02c",
)
mod = ExponentialModel()
pars = mod.guess(group.amp, x=group.vclist)
fit = mod.fit(group.amp, x=group.vclist, params=pars)
sim_x = np.linspace(group.vclist.min(), group.vclist.max())
sim_y = fit.eval(x=sim_x)
plt.plot(sim_x, sim_y, "-", c="#1f78b4", linewidth=4)
plot = plt.gca()
plot.axes.xaxis.set_ticklabels([])
plot.axes.yaxis.set_ticklabels([])
for axis in ["top", "bottom", "left", "right"]:
plot.axes.spines[axis].set_linewidth(2.0)
# plt.legend()
plt.savefig("eg_fit.pdf")
plt.show()
exit()
