def estimateNoise(data, method='MAD'):
if (method == "MAD"):
noise = mad(data)
print(noise)
return noise
def fit_plot(self,objname,N=10):
print('getting data')
crvdat = get_data(objname,bands=self.fltnames)
print(str(len(crvdat))+' data points')
print('Getting period')
#plist = get_periods(crvdat['mjd'],crvdat['mag'],crvdat['err'],crvdat['fltr'],
# objname=objname,bands=self.fltnames,N=10)
#print('periods: ',plist)
period = 0.60527109
self.selftemplate(crvdat,period)
# Fit curve
print('First fitting')
pars,p,err,tmpind,chi2 = self.tmpfit(crvdat['mjd'],crvdat['mag'],crvdat['err'],crvdat['fltr'],plist)
# Reject outliers, select inliers
resid = np.array(abs(crvdat['mag']-self.model(crvdat['mjd'],*pars)))
crvdat['inlier'] = resid<utils.mad(resid)*5
print(str(len(np.sum(~crvdat['inlier'])))+' outliers rejected')
# Fit with inliers only
print('Second fitting')
pars,p,err,tmpind,chi2 = self.tmpfit(crvdat['mjd'][crvdat['inlier']],crvdat['mag'][crvdat['inlier']],
crvdat['err'][crvdat['inlier']],crvdat['fltr'][crvdat['inlier']],plist,pars)
redchi2 = chi2/(sum(crvdat['inlier'])-len(set(crvdat['fltr'][crvdat['inlier']]))-2)
# get the filters with inlier data (incase it's different from all data)
inlierflts = set(crvdat['fltr'][crvdat['inlier']])
# Add phase to crvdat and sort
crvdat['ph'] = ph = (crvdat['mjd'] - pars[0]) / p %1
crvdat.sort(['fltr','ph'])
self.fltinds = crvdat['fltr']
# Plot
colors = ['#1f77b4','#2ca02c','#d62728','#9467bd','#8c564b','y','k']
nf = len(inlierflts) # Number of filters with inliers
fig, ax = plt.subplots(nf, figsize=(12,4*(nf**.75+1)), sharex=True)
if nf == 1:
ax = [ax]
for i,f in enumerate(inlierflts):
sel = crvdat['fltr'] == f
ax[i].scatter(crvdat['ph'][sel],crvdat['mag'][sel],c=colors[f])
ax[i].scatter(crvdat['ph'][sel]+1,crvdat['mag'][sel],c=colors[f])
tmpmag = np.tile(self.tmps.columns[tmpind]*pars[1]*self.ampratio[f]+pars[2:][f],2)
tmpph = np.tile(self.tmps['PH'],2)+([0]*len(self.tmps['PH'])+[1]*len(self.tmps['PH']))
ax[i].plot(tmpph,tmpmag,c='k')
ax[i].invert_yaxis()
ax[i].set_ylabel(self.fltnames[f], fontsize=20)
ax[-1].set_xlabel('Phase', fontsize=20)
ax[0].set_title("Object: {} Period: {:.3f} d Type: {}".format(
objname,p,self.tmps.colnames[tmpind]), fontsize=22)
fig.savefig('results/plots/{}_plot.png'.format(objname))
plt.close(fig)
# save parameters and results
res = Table([[objname]],names=['name'])
res['period'] = p
res['t0'] = pars[0]
res['r amp'] = pars[1]
for i in range(2,len(pars)):
f = self.fltnames[i-2]
res['{} mag'.format(f)] = pars[i]
res['chi2'] = chi2
res['redchi2']= redchi2
res['template']= self.tmps.colnames[tmpind]
res['t0 err'] = err[0]
res['amp err'] = err[1]
for i in range(2,len(err)):
f = self.fltnames[i-2]
res['{} mag err'.format(f)] = err[i]
res['Ndat'] = len(crvdat)
res['N inliers'] = sum(crvdat['inlier'])
for i in range(len(self.fltnames)):
f = self.fltnames[i]
res['N {}'.format(f)] = sum(crvdat['fltr'][crvdat['inlier']]==i)
res.write('results/{}_res.fits'.format(objname),format='fits',overwrite=True)
return
def fit_plot(fitter, objname, plist=None, N=10, verbose=False):
if verbose:
print('Get data')
crvdat = get_data(objname, bands=fitter.fltnames)
if plist is None:
if verbose:
print('Get Periods')
ps, psi, inds = get_periods(crvdat['mjd'],
crvdat['mag'],
crvdat['err'],
crvdat['fltr'],
objname=objname,
bands=fitter.fltnames,
N=N)
plist = ps[inds]
if verbose:
print('First Fit')
# Fit curve
pars, p, err, tmpind, chi2 = fitter.tmpfit(crvdat['mjd'],
crvdat['mag'],
crvdat['err'],
crvdat['fltr'],
plist,
verbose=verbose)
if verbose:
print('Outlier Rejection')
# Reject outliers, select inliers
resid = np.array(crvdat['mag'] - fitter.model(crvdat['mjd'], *pars))
crvdat['inlier'] = abs(resid) < utils.mad(resid) * 5
if verbose:
print('Second Fit')
# Fit with inliers only
pars, p, err, tmpind, chi2 = fitter.tmpfit(
crvdat['mjd'][crvdat['inlier']],
crvdat['mag'][crvdat['inlier']],
crvdat['err'][crvdat['inlier']],
crvdat['fltr'][crvdat['inlier']],
plist,
pars,
verbose=verbose)
redchi2 = chi2 / (sum(crvdat['inlier']) -
len(set(crvdat['fltr'][crvdat['inlier']])) - 2)
# get the filters with inlier data (incase it's different from all data)
inlierflts = set(crvdat['fltr'][crvdat['inlier']])
# Add phase to crvdat and sort
crvdat['ph'] = (crvdat['mjd'] - pars[0]) / p % 1
crvdat.sort(['fltr', 'ph'])
fitter.fltinds = crvdat['fltr']
if verbose:
print('Start Plotting')
# Plot
colors = [
'#1f77b4', '#2ca02c', '#d62728', '#9467bd', '#8c564b', 'y', '#ff7f0e'
]
nf = len(inlierflts) # Number of filters with inliers
fig, ax = plt.subplots(nf, figsize=(12, 4 * (nf**.75 + 1)), sharex=True)
if nf == 1:
ax = [ax]
for i, f in enumerate(inlierflts):
sel = crvdat['fltr'] == f
ax[i].scatter(crvdat['ph'][sel], crvdat['mag'][sel], c=colors[f])
ax[i].scatter(crvdat['ph'][sel] + 1, crvdat['mag'][sel], c=colors[f])
tmpmag = np.tile(
fitter.tmps.columns[tmpind] * pars[1 + f] +
pars[1 + fitter.Nflts + f], 2)
tmpph = np.tile(fitter.tmps['PH'], 2) + ([0] * len(fitter.tmps['PH']) +
[1] * len(fitter.tmps['PH']))
ax[i].plot(tmpph, tmpmag, c='k')
xsel = sel * (~crvdat['inlier'])
ax[i].scatter(crvdat['ph'][xsel],
crvdat['mag'][xsel],
c='k',
marker='x')
ax[i].scatter(crvdat['ph'][xsel] + 1,
crvdat['mag'][xsel],
c='k',
marker='x')
ax[i].invert_yaxis()
ax[i].set_ylabel(fitter.fltnames[f], fontsize=20)
ax[-1].set_xlabel('Phase', fontsize=20)
ax[0].set_title("Object: {} Period: {:.3f} d Type: {}".format(
objname, p, fitter.tmps.colnames[tmpind]),
fontsize=22)
path = 'results/plots/{}_plot.png'.format(objname)
fig.savefig(path)
if verbose:
print('Saved to', path)
print('Save parameters')
plt.close(fig)
# save parameters and results
res = Table([[objname]], names=['name'])
res['period'] = p
res['t0'] = pars[0]
res['t0 err'] = err[0]
for i in range(fitter.Nflts):
f = fitter.fltnames[i]
res['{} amp'.format(f)] = pars[1 + i]
res['{} amp err'.format(f)] = err[1 + i]
res['{} mag'.format(f)] = pars[1 + fitter.Nflts + i]
res['{} mag err'.format(f)] = err[1 + fitter.Nflts + i]
res['chi2'] = chi2
res['redchi2'] = redchi2
res['template'] = fitter.tmps.colnames[tmpind]
res['Ndat'] = len(crvdat)
res['N outliers'] = len(crvdat) - sum(crvdat['inlier'])
for i in range(fitter.Nflts):
f = fitter.fltnames[i]
res['N {}'.format(f)] = sum(crvdat['fltr'][crvdat['inlier']] == i)
path = 'results/{}_res.fits'.format(objname)
res.write(path, format='fits', overwrite=True)
if verbose:
print('Saved to', path)
print('end')
return
def estimateNoise(data, method='MAD'):
if (method=="MAD"):
noise = mad(data)
print(noise)
return noise
def constraints_s(self,
doThr,
K,
doRw,
nnegS,
Sfft=None,
Sfft_det=None,
S=None,
stds=None,
Swtrw=None,
oracle=False):
"""Apply the constraints on the sources (thresholding in the wavelet domain and possibly a projection on the
positive orthant. The input data are Sfft. The output data are S, as well as Sfft and Sfft_det.
Parameters
----------
doThr : bool
perform thresholding
K: float
L0 support of the sources
doRw: bool
do reweighting
nnegS: bool
apply non-negativity constraint on the sources
Sfft: np.ndarray
(n,p) complex array, estimated sources in Fourier domain (in-place update, default: self.Sfft)
Sfft_det: np.ndarray
(n,p) complex array, estimated sources with only the detail scales in Fourier domain (in-place update,
default: self.Sfft_det)
S: np.ndarray
(n,p) float array, estimated sources (in-place update, default: self.S)
stds: np.ndarray
(n,nscales) float array, std of the noise in the source space, per detail scale (default: mad or analytical
calculation)
Swtrw: np.ndarray
(n,p,nscales) float array, sources in the wavelet domain of previous iteration (default: self.Swtrw)
oracle: bool
perform an oracle update (using the ground-truth A and S)
Returns
-------
int
error code
"""
if Sfft is None:
Sfft = self.Sfft
if Sfft_det is None:
Sfft_det = self.Sfft_det
if S is None:
S = self.S
if Swtrw is None:
if not oracle:
Swtrw = self.Swtrw
else:
Swtrw = self.S0wt
if not doThr:
S[:] = fftt.ifft(Sfft)
if not nnegS: # nothing more to do
Sfft_det[:] = fftt.fftprod(Sfft, 1 - self.wt_filters[:, -1])
return 0
else:
if self.verb >= 3:
print("Maximal L0 norm of the sources: %.1f %%" % (K * 100))
Swt = fftt.wt_trans(Sfft, nscales=self.nscales, fft_in=True)
# Thresholding
for i in range(self.n):
for j in range(self.nscales):
Swtij = Swt[i, :, j]
Swtrwij = Swtrw[i, :, j]
if stds is not None:
std = stds[i, j]
elif self.useMad:
std = utils.mad(Swtij, M=self.M)
else:
std = self.nStd * np.sqrt(
np.sum(self.invOpSp[:, i] *
self.wt_filters[:, j]**2) / self.p)
thrd = self.k * std
# If oracle, threshold Swtrw
if oracle and self.L1 and doRw:
Swtrwij = (
Swtrwij - np.sign(Swtrwij) * (thrd - np.sqrt(
np.abs(
(Swtrwij - thrd * np.sign(Swtrwij)) *
(3 * thrd * np.sign(Swtrwij) + Swtrwij))))
) / 2 * (np.abs(Swtrwij) >= thrd)
# Support based threshold
if K != 1:
npix = np.sum(abs(Swtij) - thrd > 0)
Kval = np.maximum(np.int(K * npix), 5)
thrd = np.partition(abs(Swtij),
self.p - Kval)[self.p - Kval]
if self.verb == 4 and i == 0:
print("Threshold of source %i at scale %i: %.5e" %
(i + 1, j + 1, thrd))
elif self.verb == 5:
print("Threshold of source %i at scale %i: %.5e" %
(i + 1, j + 1, thrd))
# Adapt the threshold if reweighing demanded
if doRw and self.L1:
thrd = thrd / (np.abs(Swtrwij) /
np.maximum(1e-20, self.k * std) + 1)
else:
thrd = thrd * np.ones(self.p)
# Apply the threshold
Swtij[(abs(Swtij) < thrd)] = 0
if self.L1:
indNZ = np.where(abs(Swtij) > thrd)[0]
Swtij[indNZ] = Swtij[indNZ] - thrd[indNZ] * np.sign(
Swtij[indNZ])
Swt[i, :, j] = Swtij
# Reconstruct S
S[:] = fftt.wt_rec(Swt)
# Non-negativity constraint
if nnegS:
nneg = S > 0
S *= nneg
if oracle:
return 0
# Save the wavelet coefficients of S for next iteration
if doThr and doRw and self.L1 and not oracle:
if nnegS:
Swt *= nneg[:, :, np.newaxis]
self.Swtrw = Swt[:, :, :-1]
Sfft[:] = fftt.fft(S)
Sfft_det[:] = fftt.fftprod(Sfft, 1 - self.wt_filters[:, -1])
return 0
lna_average = np.asarray(
[np.median(lna[idx == ivar]) for ivar in range(len(gaovla_average))])
ndo_average = np.asarray(
[np.median(ndo[idx == ivar]) for ivar in range(len(gaovla_average))])
fdo_average = np.asarray(
[np.median(fdo[idx == ivar]) for ivar in range(len(gaovla_average))])
ldo_average = np.asarray(
[np.median(ldo[idx == ivar]) for ivar in range(len(gaovla_average))])
exp_average = np.asarray(
[np.median(export[idx == ivar]) for ivar in range(len(gaovla_average))])
exp1_average = np.asarray(
[np.median(exp1[idx == ivar]) for ivar in range(len(gaovla_average))])
exp2_average = np.asarray(
[np.median(exp2[idx == ivar]) for ivar in range(len(gaovla_average))])
jovern_spread = np.asarray(
[utils.mad(jovern[idx == ivar]) for ivar in range(len(gaovla_average))])
jovernn_spread = np.asarray(
[utils.mad(jovernn[idx == ivar]) for ivar in range(len(gaovla_average))])
jovernf_spread = np.asarray(
[utils.mad(jovernf[idx == ivar]) for ivar in range(len(gaovla_average))])
jovernl_spread = np.asarray(
[utils.mad(jovernl[idx == ivar]) for ivar in range(len(gaovla_average))])
pstar_spread = np.asarray(
[utils.mad(pstar[idx == ivar]) for ivar in range(len(gaovla_average))])
nsurf_spread = np.asarray(
[utils.mad(nsurfmean[idx == ivar]) for ivar in range(len(gaovla_average))])
fsurf_spread = np.asarray(
[utils.mad(fsurfmean[idx == ivar]) for ivar in range(len(gaovla_average))])
lsurf_spread = np.asarray(
[utils.mad(lsurfmean[idx == ivar]) for ivar in range(len(gaovla_average))])
exp_spread = np.asarray(
y_svc = np.array([np.log10(y) if y != 0 else 0 for y in y_svc]).ravel()
y_tree = np.array([np.log10(y) if y != 0 else 0 for y in y_tree]).ravel()
y_nb = np.array([np.log10(y) if y != 0 else 0 for y in y_nb]).ravel()
y_boosting = np.array([np.log10(y) if y != 0 else 0
for y in y_boosting]).ravel()
y_lr = np.array([np.log10(y) if y != 0 else 0 for y in y_lr]).ravel()
y_knn = np.array([np.log10(y) if y != 0 else 0 for y in y_knn]).ravel()
classifiers = [y_rf, y_svc, y_tree, y_nb, y_boosting, y_lr, y_knn]
mad_values = np.empty(shape=(5, len(classifiers)))
rmse_values = np.empty(shape=(5, len(classifiers)))
# Evaluate final regression
for j, target in enumerate(classifiers):
median_estimates = baseline_prediction(target)
mad_values[0, j] = mad(median_estimates, target)
rmse_values[0, j] = rmse(median_estimates, target)
mean_estimates = baseline_prediction(target, strategy='mean')
mad_values[1, j] = mad(mean_estimates, target)
rmse_values[1, j] = rmse(mean_estimates, target)
rr_estimates = regressor_prediction(X, target, Ridge)
mad_values[2, j] = mad(rr_estimates, target)
rmse_values[2, j] = rmse(rr_estimates, target)
rf_estimates = regressor_prediction(X, target, RandomForestRegressor)
mad_values[3, j] = mad(rf_estimates, target)
rmse_values[3, j] = rmse(rf_estimates, target)
svr_estimates = regressor_prediction(X, target, SVR)