def estimateNoise(data, method='MAD'): if (method == "MAD"): noise = mad(data) print(noise) return noise
def fit_plot(self,objname,N=10): print('getting data') crvdat = get_data(objname,bands=self.fltnames) print(str(len(crvdat))+' data points') print('Getting period') #plist = get_periods(crvdat['mjd'],crvdat['mag'],crvdat['err'],crvdat['fltr'], # objname=objname,bands=self.fltnames,N=10) #print('periods: ',plist) period = 0.60527109 self.selftemplate(crvdat,period) # Fit curve print('First fitting') pars,p,err,tmpind,chi2 = self.tmpfit(crvdat['mjd'],crvdat['mag'],crvdat['err'],crvdat['fltr'],plist) # Reject outliers, select inliers resid = np.array(abs(crvdat['mag']-self.model(crvdat['mjd'],*pars))) crvdat['inlier'] = resid<utils.mad(resid)*5 print(str(len(np.sum(~crvdat['inlier'])))+' outliers rejected') # Fit with inliers only print('Second fitting') pars,p,err,tmpind,chi2 = self.tmpfit(crvdat['mjd'][crvdat['inlier']],crvdat['mag'][crvdat['inlier']], crvdat['err'][crvdat['inlier']],crvdat['fltr'][crvdat['inlier']],plist,pars) redchi2 = chi2/(sum(crvdat['inlier'])-len(set(crvdat['fltr'][crvdat['inlier']]))-2) # get the filters with inlier data (incase it's different from all data) inlierflts = set(crvdat['fltr'][crvdat['inlier']]) # Add phase to crvdat and sort crvdat['ph'] = ph = (crvdat['mjd'] - pars[0]) / p %1 crvdat.sort(['fltr','ph']) self.fltinds = crvdat['fltr'] # Plot colors = ['#1f77b4','#2ca02c','#d62728','#9467bd','#8c564b','y','k'] nf = len(inlierflts) # Number of filters with inliers fig, ax = plt.subplots(nf, figsize=(12,4*(nf**.75+1)), sharex=True) if nf == 1: ax = [ax] for i,f in enumerate(inlierflts): sel = crvdat['fltr'] == f ax[i].scatter(crvdat['ph'][sel],crvdat['mag'][sel],c=colors[f]) ax[i].scatter(crvdat['ph'][sel]+1,crvdat['mag'][sel],c=colors[f]) tmpmag = np.tile(self.tmps.columns[tmpind]*pars[1]*self.ampratio[f]+pars[2:][f],2) tmpph = np.tile(self.tmps['PH'],2)+([0]*len(self.tmps['PH'])+[1]*len(self.tmps['PH'])) ax[i].plot(tmpph,tmpmag,c='k') ax[i].invert_yaxis() ax[i].set_ylabel(self.fltnames[f], fontsize=20) ax[-1].set_xlabel('Phase', fontsize=20) ax[0].set_title("Object: {} Period: {:.3f} d Type: {}".format( objname,p,self.tmps.colnames[tmpind]), fontsize=22) fig.savefig('results/plots/{}_plot.png'.format(objname)) plt.close(fig) # save parameters and results res = Table([[objname]],names=['name']) res['period'] = p res['t0'] = pars[0] res['r amp'] = pars[1] for i in range(2,len(pars)): f = self.fltnames[i-2] res['{} mag'.format(f)] = pars[i] res['chi2'] = chi2 res['redchi2']= redchi2 res['template']= self.tmps.colnames[tmpind] res['t0 err'] = err[0] res['amp err'] = err[1] for i in range(2,len(err)): f = self.fltnames[i-2] res['{} mag err'.format(f)] = err[i] res['Ndat'] = len(crvdat) res['N inliers'] = sum(crvdat['inlier']) for i in range(len(self.fltnames)): f = self.fltnames[i] res['N {}'.format(f)] = sum(crvdat['fltr'][crvdat['inlier']]==i) res.write('results/{}_res.fits'.format(objname),format='fits',overwrite=True) return
def fit_plot(fitter, objname, plist=None, N=10, verbose=False): if verbose: print('Get data') crvdat = get_data(objname, bands=fitter.fltnames) if plist is None: if verbose: print('Get Periods') ps, psi, inds = get_periods(crvdat['mjd'], crvdat['mag'], crvdat['err'], crvdat['fltr'], objname=objname, bands=fitter.fltnames, N=N) plist = ps[inds] if verbose: print('First Fit') # Fit curve pars, p, err, tmpind, chi2 = fitter.tmpfit(crvdat['mjd'], crvdat['mag'], crvdat['err'], crvdat['fltr'], plist, verbose=verbose) if verbose: print('Outlier Rejection') # Reject outliers, select inliers resid = np.array(crvdat['mag'] - fitter.model(crvdat['mjd'], *pars)) crvdat['inlier'] = abs(resid) < utils.mad(resid) * 5 if verbose: print('Second Fit') # Fit with inliers only pars, p, err, tmpind, chi2 = fitter.tmpfit( crvdat['mjd'][crvdat['inlier']], crvdat['mag'][crvdat['inlier']], crvdat['err'][crvdat['inlier']], crvdat['fltr'][crvdat['inlier']], plist, pars, verbose=verbose) redchi2 = chi2 / (sum(crvdat['inlier']) - len(set(crvdat['fltr'][crvdat['inlier']])) - 2) # get the filters with inlier data (incase it's different from all data) inlierflts = set(crvdat['fltr'][crvdat['inlier']]) # Add phase to crvdat and sort crvdat['ph'] = (crvdat['mjd'] - pars[0]) / p % 1 crvdat.sort(['fltr', 'ph']) fitter.fltinds = crvdat['fltr'] if verbose: print('Start Plotting') # Plot colors = [ '#1f77b4', '#2ca02c', '#d62728', '#9467bd', '#8c564b', 'y', '#ff7f0e' ] nf = len(inlierflts) # Number of filters with inliers fig, ax = plt.subplots(nf, figsize=(12, 4 * (nf**.75 + 1)), sharex=True) if nf == 1: ax = [ax] for i, f in enumerate(inlierflts): sel = crvdat['fltr'] == f ax[i].scatter(crvdat['ph'][sel], crvdat['mag'][sel], c=colors[f]) ax[i].scatter(crvdat['ph'][sel] + 1, crvdat['mag'][sel], c=colors[f]) tmpmag = np.tile( fitter.tmps.columns[tmpind] * pars[1 + f] + pars[1 + fitter.Nflts + f], 2) tmpph = np.tile(fitter.tmps['PH'], 2) + ([0] * len(fitter.tmps['PH']) + [1] * len(fitter.tmps['PH'])) ax[i].plot(tmpph, tmpmag, c='k') xsel = sel * (~crvdat['inlier']) ax[i].scatter(crvdat['ph'][xsel], crvdat['mag'][xsel], c='k', marker='x') ax[i].scatter(crvdat['ph'][xsel] + 1, crvdat['mag'][xsel], c='k', marker='x') ax[i].invert_yaxis() ax[i].set_ylabel(fitter.fltnames[f], fontsize=20) ax[-1].set_xlabel('Phase', fontsize=20) ax[0].set_title("Object: {} Period: {:.3f} d Type: {}".format( objname, p, fitter.tmps.colnames[tmpind]), fontsize=22) path = 'results/plots/{}_plot.png'.format(objname) fig.savefig(path) if verbose: print('Saved to', path) print('Save parameters') plt.close(fig) # save parameters and results res = Table([[objname]], names=['name']) res['period'] = p res['t0'] = pars[0] res['t0 err'] = err[0] for i in range(fitter.Nflts): f = fitter.fltnames[i] res['{} amp'.format(f)] = pars[1 + i] res['{} amp err'.format(f)] = err[1 + i] res['{} mag'.format(f)] = pars[1 + fitter.Nflts + i] res['{} mag err'.format(f)] = err[1 + fitter.Nflts + i] res['chi2'] = chi2 res['redchi2'] = redchi2 res['template'] = fitter.tmps.colnames[tmpind] res['Ndat'] = len(crvdat) res['N outliers'] = len(crvdat) - sum(crvdat['inlier']) for i in range(fitter.Nflts): f = fitter.fltnames[i] res['N {}'.format(f)] = sum(crvdat['fltr'][crvdat['inlier']] == i) path = 'results/{}_res.fits'.format(objname) res.write(path, format='fits', overwrite=True) if verbose: print('Saved to', path) print('end') return
def estimateNoise(data, method='MAD'): if (method=="MAD"): noise = mad(data) print(noise) return noise
def constraints_s(self, doThr, K, doRw, nnegS, Sfft=None, Sfft_det=None, S=None, stds=None, Swtrw=None, oracle=False): """Apply the constraints on the sources (thresholding in the wavelet domain and possibly a projection on the positive orthant. The input data are Sfft. The output data are S, as well as Sfft and Sfft_det. Parameters ---------- doThr : bool perform thresholding K: float L0 support of the sources doRw: bool do reweighting nnegS: bool apply non-negativity constraint on the sources Sfft: np.ndarray (n,p) complex array, estimated sources in Fourier domain (in-place update, default: self.Sfft) Sfft_det: np.ndarray (n,p) complex array, estimated sources with only the detail scales in Fourier domain (in-place update, default: self.Sfft_det) S: np.ndarray (n,p) float array, estimated sources (in-place update, default: self.S) stds: np.ndarray (n,nscales) float array, std of the noise in the source space, per detail scale (default: mad or analytical calculation) Swtrw: np.ndarray (n,p,nscales) float array, sources in the wavelet domain of previous iteration (default: self.Swtrw) oracle: bool perform an oracle update (using the ground-truth A and S) Returns ------- int error code """ if Sfft is None: Sfft = self.Sfft if Sfft_det is None: Sfft_det = self.Sfft_det if S is None: S = self.S if Swtrw is None: if not oracle: Swtrw = self.Swtrw else: Swtrw = self.S0wt if not doThr: S[:] = fftt.ifft(Sfft) if not nnegS: # nothing more to do Sfft_det[:] = fftt.fftprod(Sfft, 1 - self.wt_filters[:, -1]) return 0 else: if self.verb >= 3: print("Maximal L0 norm of the sources: %.1f %%" % (K * 100)) Swt = fftt.wt_trans(Sfft, nscales=self.nscales, fft_in=True) # Thresholding for i in range(self.n): for j in range(self.nscales): Swtij = Swt[i, :, j] Swtrwij = Swtrw[i, :, j] if stds is not None: std = stds[i, j] elif self.useMad: std = utils.mad(Swtij, M=self.M) else: std = self.nStd * np.sqrt( np.sum(self.invOpSp[:, i] * self.wt_filters[:, j]**2) / self.p) thrd = self.k * std # If oracle, threshold Swtrw if oracle and self.L1 and doRw: Swtrwij = ( Swtrwij - np.sign(Swtrwij) * (thrd - np.sqrt( np.abs( (Swtrwij - thrd * np.sign(Swtrwij)) * (3 * thrd * np.sign(Swtrwij) + Swtrwij)))) ) / 2 * (np.abs(Swtrwij) >= thrd) # Support based threshold if K != 1: npix = np.sum(abs(Swtij) - thrd > 0) Kval = np.maximum(np.int(K * npix), 5) thrd = np.partition(abs(Swtij), self.p - Kval)[self.p - Kval] if self.verb == 4 and i == 0: print("Threshold of source %i at scale %i: %.5e" % (i + 1, j + 1, thrd)) elif self.verb == 5: print("Threshold of source %i at scale %i: %.5e" % (i + 1, j + 1, thrd)) # Adapt the threshold if reweighing demanded if doRw and self.L1: thrd = thrd / (np.abs(Swtrwij) / np.maximum(1e-20, self.k * std) + 1) else: thrd = thrd * np.ones(self.p) # Apply the threshold Swtij[(abs(Swtij) < thrd)] = 0 if self.L1: indNZ = np.where(abs(Swtij) > thrd)[0] Swtij[indNZ] = Swtij[indNZ] - thrd[indNZ] * np.sign( Swtij[indNZ]) Swt[i, :, j] = Swtij # Reconstruct S S[:] = fftt.wt_rec(Swt) # Non-negativity constraint if nnegS: nneg = S > 0 S *= nneg if oracle: return 0 # Save the wavelet coefficients of S for next iteration if doThr and doRw and self.L1 and not oracle: if nnegS: Swt *= nneg[:, :, np.newaxis] self.Swtrw = Swt[:, :, :-1] Sfft[:] = fftt.fft(S) Sfft_det[:] = fftt.fftprod(Sfft, 1 - self.wt_filters[:, -1]) return 0
lna_average = np.asarray( [np.median(lna[idx == ivar]) for ivar in range(len(gaovla_average))]) ndo_average = np.asarray( [np.median(ndo[idx == ivar]) for ivar in range(len(gaovla_average))]) fdo_average = np.asarray( [np.median(fdo[idx == ivar]) for ivar in range(len(gaovla_average))]) ldo_average = np.asarray( [np.median(ldo[idx == ivar]) for ivar in range(len(gaovla_average))]) exp_average = np.asarray( [np.median(export[idx == ivar]) for ivar in range(len(gaovla_average))]) exp1_average = np.asarray( [np.median(exp1[idx == ivar]) for ivar in range(len(gaovla_average))]) exp2_average = np.asarray( [np.median(exp2[idx == ivar]) for ivar in range(len(gaovla_average))]) jovern_spread = np.asarray( [utils.mad(jovern[idx == ivar]) for ivar in range(len(gaovla_average))]) jovernn_spread = np.asarray( [utils.mad(jovernn[idx == ivar]) for ivar in range(len(gaovla_average))]) jovernf_spread = np.asarray( [utils.mad(jovernf[idx == ivar]) for ivar in range(len(gaovla_average))]) jovernl_spread = np.asarray( [utils.mad(jovernl[idx == ivar]) for ivar in range(len(gaovla_average))]) pstar_spread = np.asarray( [utils.mad(pstar[idx == ivar]) for ivar in range(len(gaovla_average))]) nsurf_spread = np.asarray( [utils.mad(nsurfmean[idx == ivar]) for ivar in range(len(gaovla_average))]) fsurf_spread = np.asarray( [utils.mad(fsurfmean[idx == ivar]) for ivar in range(len(gaovla_average))]) lsurf_spread = np.asarray( [utils.mad(lsurfmean[idx == ivar]) for ivar in range(len(gaovla_average))]) exp_spread = np.asarray(
y_svc = np.array([np.log10(y) if y != 0 else 0 for y in y_svc]).ravel() y_tree = np.array([np.log10(y) if y != 0 else 0 for y in y_tree]).ravel() y_nb = np.array([np.log10(y) if y != 0 else 0 for y in y_nb]).ravel() y_boosting = np.array([np.log10(y) if y != 0 else 0 for y in y_boosting]).ravel() y_lr = np.array([np.log10(y) if y != 0 else 0 for y in y_lr]).ravel() y_knn = np.array([np.log10(y) if y != 0 else 0 for y in y_knn]).ravel() classifiers = [y_rf, y_svc, y_tree, y_nb, y_boosting, y_lr, y_knn] mad_values = np.empty(shape=(5, len(classifiers))) rmse_values = np.empty(shape=(5, len(classifiers))) # Evaluate final regression for j, target in enumerate(classifiers): median_estimates = baseline_prediction(target) mad_values[0, j] = mad(median_estimates, target) rmse_values[0, j] = rmse(median_estimates, target) mean_estimates = baseline_prediction(target, strategy='mean') mad_values[1, j] = mad(mean_estimates, target) rmse_values[1, j] = rmse(mean_estimates, target) rr_estimates = regressor_prediction(X, target, Ridge) mad_values[2, j] = mad(rr_estimates, target) rmse_values[2, j] = rmse(rr_estimates, target) rf_estimates = regressor_prediction(X, target, RandomForestRegressor) mad_values[3, j] = mad(rf_estimates, target) rmse_values[3, j] = rmse(rf_estimates, target) svr_estimates = regressor_prediction(X, target, SVR)