def simple_spectrum(obs): """Simple estimate of spectrum Parameters ---------- obs : shadow.Observation Input observation Returns ------- m_spat : np.ndarray Array of the mean spectrum g_spat : np.ndarray Array of the gradient of the spectrum """ #Cut out dat = np.copy( (obs.sci / obs.flat))[:, obs.spatial.reshape(-1)][:, :, obs.spectral.reshape(-1)] err = np.copy( (obs.err / obs.flat))[:, obs.spatial.reshape(-1)][:, :, obs.spectral.reshape(-1)] dat /= obs.vsr_mean err /= obs.vsr_mean m_spec = np.average(dat[~obs.in_transit][(~obs.in_transit).sum() // 2], weights=1 / err[~obs.in_transit][(~obs.in_transit).sum() // 2], axis=0) m_spec /= np.mean(m_spec) x = np.linspace(0.5, -0.5, dat.shape[2]) x_p = np.linspace(-0.5, 0.5, dat.shape[2] * 10) l = lk.LightCurve(x, m_spec) l = l.normalize() r = lk.RegressionCorrector(l) dm = lk.designmatrix.create_spline_matrix( x, knots=list(np.linspace(-0.5, 0.5, int( dat.shape[2] / 1.5))[1:-1])).append_constant() dm_p = lk.designmatrix.create_spline_matrix( x_p, knots=list(np.linspace(-0.5, 0.5, int( dat.shape[2] / 1.5))[1:-1])).append_constant() _ = r.correct(dm, sigma=1e10) model = dm_p.X.dot(r.coefficients) g_spec = np.gradient(model, x_p)[::10] m_spec = (np.atleast_3d(m_spec) * np.ones(dat.shape).transpose([0, 2, 1])).transpose([0, 2, 1]) return m_spec, g_spec
def find_number_of_PCs(results, regressors, lc): ''' Purpose: Find the number of principal components to use in the PCA method ''' # Parameters / Criteria npc = 7 nbins = 40 # number of bins to divide the each PC threshold_variance = 1e-4 # 5 # -> 500% # THIS IS NOT USED. dm = lk.DesignMatrix(regressors, name='regressors').pca(npc).append_constant() rc = lk.RegressionCorrector(lc) lc_regressed = rc.correct(dm) # Find the median of the moving variance for each PCs boxsize = np.floor(dm.values[:, 0].size / nbins).astype(int) pcs_variances = [] for ipc in range(npc): pcs_variances.append( pd.Series(dm.values[:, ipc]).rolling(boxsize).var().median()) # pcs_variances = [ pd.Series(dm.values[:,ipc]).rolling(boxsize).var().median() for ipc in range(npc) ] pcs_variances = np.array(pcs_variances) relative_change_variances = np.abs( np.diff(pcs_variances) / pcs_variances[:-1]) # < THIS IS NOT USED. # Select PCs with variance above threshold value ind_bad_pcs = np.argwhere(pcs_variances > threshold_variance) if ind_bad_pcs.size > 0: # Find index first PC that exceeds threshold value. This is the new npc new_npc = ind_bad_pcs[0].item() else: print( f'No PCs with variance>{threshold_variance}. All {npc} PCs used.') new_npc = npc # Store to results results['pca_all'] = {'coef':rc.coefficients,\ 'pc':[dm.values[:,i] for i in range(dm.rank)],\ 'dm':dm,\ 'rc':rc,\ 'npc':npc,\ 'npc_used':new_npc,\ 'pc_variances':pcs_variances,\ 'threshold_variance':threshold_variance,\ 'nbins':nbins} return new_npc, dm, rc
def build(self, object_info: MissionObjectInfo, sherlock_dir, caches_root_dir): mission_id = object_info.mission_id() sherlock_id = object_info.sherlock_id() logging.info("Retrieving star catalog info...") mission, mission_prefix, id = super().parse_object_id(mission_id) if mission_prefix not in self.star_catalogs: raise ValueError("Wrong object id " + mission_id) cadence = object_info.cadence if object_info.cadence is not None else "short" author = object_info.author if object_info.author is not None else self.authors[mission] star_info = starinfo.StarInfo(sherlock_id, *self.star_catalogs[mission_prefix].catalog_info(id)) logging.info("Downloading lightcurve files...") sectors = None if object_info.sectors == 'all' or mission != constants.MISSION_TESS else object_info.sectors campaigns = None if object_info.sectors == 'all' or mission != constants.MISSION_K2 else object_info.sectors quarters = None if object_info.sectors == 'all' or mission != constants.MISSION_KEPLER else object_info.sectors tokens = sectors if sectors is not None else campaigns if campaigns is not None else quarters tokens = tokens if tokens is not None else "all" apertures = {} tpf_search_results = lk.search_targetpixelfile(str(mission_id)) for tpf_search_result in tpf_search_results: logging.info("There is data for Mission: %s, Year %.0f, Author: %s, ExpTime: %.0f", tpf_search_result.mission[0], tpf_search_result.year[0], tpf_search_result.author[0], tpf_search_result.exptime[0].value) tpfs_dir = sherlock_dir + "/tpfs/" if not os.path.exists(tpfs_dir): os.mkdir(tpfs_dir) if object_info.apertures is None: lcf_search_results = lk.search_lightcurve(str(mission_id), mission=mission, cadence=cadence, sector=sectors, quarter=quarters, campaign=campaigns, author=author) lcf = lcf_search_results.download_all(download_dir=caches_root_dir + LIGHTKURVE_CACHE_DIR) tpfs = lk.search_targetpixelfile(str(mission_id), mission=mission, cadence=cadence, sector=sectors, quarter=quarters, campaign=campaigns, author=author)\ .download_all(download_dir=caches_root_dir + LIGHTKURVE_CACHE_DIR, cutout_size=(CUTOUT_SIZE, CUTOUT_SIZE)) if lcf is None: raise ObjectProcessingError("The target " + str(mission_id) + " is not available for the author " + author + ", cadence " + str(cadence) + "s and sectors " + str(tokens)) lc_data = self.extract_lc_data(lcf) lc = None matching_objects = [] for tpf in tpfs: shutil.copy(tpf.path, tpfs_dir + os.path.basename(tpf.path)) if mission_prefix == self.MISSION_ID_KEPLER: sector = tpf.quarter elif mission_prefix == self.MISSION_ID_TESS: sector = tpf.sector if mission_prefix == self.MISSION_ID_KEPLER_2: sector = tpf.campaign apertures[sector] = ApertureExtractor.from_boolean_mask(tpf.pipeline_mask, tpf.column, tpf.row) for i in range(0, len(lcf.data)): if lcf.data[i].label == mission_id: if lc is None: lc = lcf.data[i].normalize() else: lc = lc.append(lcf.data[i].normalize()) else: matching_objects.append(lcf.data[i].label) matching_objects = set(matching_objects) if len(matching_objects) > 0: logging.warning("================================================") logging.warning("TICS IN THE SAME PIXEL: " + str(matching_objects)) logging.warning("================================================") if lc is None: tokens = sectors if sectors is not None else campaigns if campaigns is not None else quarters tokens = tokens if tokens is not None else "all" raise ObjectProcessingError("The target " + str(mission_id) + " is not available for the author " + author + ", cadence " + str(cadence) + "s and sectors " + str(tokens)) lc = lc.remove_nans() transits_min_count = self.__calculate_transits_min_count(len(lcf)) if mission_prefix == self.MISSION_ID_KEPLER: sectors = [lcfile.quarter for lcfile in lcf] elif mission_prefix == self.MISSION_ID_TESS: sectors = [file.sector for file in lcf] elif mission_prefix == self.MISSION_ID_KEPLER_2: logging.info("Correcting K2 motion in light curve...") sectors = [lcfile.campaign for lcfile in lcf] lc = lc.to_corrector("sff").correct(windows=20) source = "tpf" else: logging.info("Using user apertures!") tpf_search_results = lk.search_targetpixelfile(str(mission_id), mission=mission, cadence=cadence, sector=sectors, quarter=quarters, campaign=campaigns, author=author) tpfs = tpf_search_results.download_all(download_dir=caches_root_dir + LIGHTKURVE_CACHE_DIR, cutout_size=(CUTOUT_SIZE, CUTOUT_SIZE)) source = "tpf" apertures = object_info.apertures lc = None for tpf in tpfs: shutil.copy(tpf.path, tpfs_dir + os.path.basename(tpf.path)) if mission_prefix == self.MISSION_ID_KEPLER: sector = tpf.quarter elif mission_prefix == self.MISSION_ID_TESS: sector = tpf.sector elif mission_prefix == self.MISSION_ID_KEPLER_2: sector = tpf.campaign boolean_aperture = ApertureExtractor.from_pixels_to_boolean_mask(apertures[sector], tpf.column, tpf.row, CUTOUT_SIZE, CUTOUT_SIZE) tpf.plot(aperture_mask=boolean_aperture, mask_color='red') plt.savefig(sherlock_dir + "/fov/Aperture_[" + str(sector) + "].png") plt.close() if mission_prefix == self.MISSION_ID_KEPLER: corrector = lk.KeplerCBVCorrector(tpf) corrector.plot_cbvs([1, 2, 3, 4, 5, 6, 7]) raw_lc = tpf.to_lightcurve(aperture_mask=boolean_aperture).remove_nans() plt.savefig(sherlock_dir + "/Corrector_components[" + str(sector) + "].png") plt.close() it_lc = corrector.correct([1, 2, 3, 4, 5]) ax = raw_lc.plot(color='C3', label='SAP Flux', linestyle='-') it_lc.plot(ax=ax, color='C2', label='CBV Corrected SAP Flux', linestyle='-') plt.savefig(sherlock_dir + "/Raw_vs_CBVcorrected_lc[" + str(sector) + "].png") plt.close() elif mission_prefix == self.MISSION_ID_KEPLER_2: raw_lc = tpf.to_lightcurve(aperture_mask=boolean_aperture).remove_nans() it_lc = raw_lc.to_corrector("sff").correct(windows=20) ax = raw_lc.plot(color='C3', label='SAP Flux', linestyle='-') it_lc.plot(ax=ax, color='C2', label='CBV Corrected SAP Flux', linestyle='-') plt.savefig(sherlock_dir + "/Raw_vs_SFFcorrected_lc[" + str(sector) + "].png") plt.close() elif mission_prefix == self.MISSION_ID_TESS: temp_lc = tpf.to_lightcurve(aperture_mask=boolean_aperture) where_are_NaNs = np.isnan(temp_lc.flux) temp_lc = temp_lc[np.where(~where_are_NaNs)] regressors = tpf.flux[np.argwhere(~where_are_NaNs), ~boolean_aperture] temp_token_lc = [temp_lc[i: i + 2000] for i in range(0, len(temp_lc), 2000)] regressors_token = [regressors[i: i + 2000] for i in range(0, len(regressors), 2000)] it_lc = None raw_it_lc = None item_index = 0 for temp_token_lc_item in temp_token_lc: regressors_token_item = regressors_token[item_index] design_matrix = lk.DesignMatrix(regressors_token_item, name='regressors').pca(5).append_constant() corr_lc = lk.RegressionCorrector(temp_token_lc_item).correct(design_matrix) if it_lc is None: it_lc = corr_lc raw_it_lc = temp_token_lc_item else: it_lc = it_lc.append(corr_lc) raw_it_lc = raw_it_lc.append(temp_token_lc_item) item_index = item_index + 1 ax = raw_it_lc.plot(label='Raw light curve') it_lc.plot(ax=ax, label='Corrected light curve') plt.savefig(sherlock_dir + "/Raw_vs_DMcorrected_lc[" + str(sector) + "].png") plt.close() if lc is None: lc = it_lc.normalize() else: lc = lc.append(it_lc.normalize()) lc = lc.remove_nans() lc.plot(label="Normalized light curve") plt.savefig(sherlock_dir + "/Normalized_lc[" + str(sector) + "].png") plt.close() transits_min_count = self.__calculate_transits_min_count(len(tpfs)) if mission_prefix == self.MISSION_ID_KEPLER or mission_id == self.MISSION_ID_KEPLER_2: sectors = [lcfile.quarter for lcfile in tpfs] elif mission_prefix == self.MISSION_ID_TESS: sectors = [file.sector for file in tpfs] if mission_prefix == self.MISSION_ID_KEPLER_2: logging.info("Correcting K2 motion in light curve...") sectors = [lcfile.campaign for lcfile in tpfs] sectors = None if sectors is None else np.unique(sectors) lc_data = None return LcBuild(lc, lc_data, star_info, transits_min_count, cadence, None, sectors, source, apertures)
def make_custom_lc( self, sector=None, sap_mask=None, aper_radius=None, percentile=None, threshold_sigma=None, use_pld=True, pixel_components=3, spline_n_knots=100, spline_degree=3, background_mask=None, pca_nterms=5, with_offset=True, ): """ create a custom lightcurve with background subtraction, based on this tutorial: https://docs.lightkurve.org/tutorials/04-how-to-remove-tess-scattered-light-using-regressioncorrector.html Parameters ---------- sector : int or str specific sector or all aper_radius: int aperture mask radius percentile: float aperture mask percentile threshold_sigma: float aperture mask threshold [sigma] pca_nterms : int number of pca terms to use Returns ------- corrected_lc : lightkurve object """ if self.verbose: print("Using lightcurve with custom aperture.") sector = sector if sector is not None else self.sector sap_mask = sap_mask if sap_mask else self.sap_mask aper_radius = aper_radius if aper_radius else self.aper_radius percentile = percentile if percentile else self.percentile threshold_sigma = (threshold_sigma if threshold_sigma else self.threshold_sigma) if self.tpf is None: tpf, tpf_info = self.get_tpf(sector=sector, return_df=True) else: if self.tpf.sector == sector: tpf = self.tpf else: tpf, tpf_info = self.get_tpf(sector=sector, return_df=True) # Make an aperture mask and a raw light curve self.aper_mask = parse_aperture_mask( tpf, sap_mask=sap_mask, aper_radius=aper_radius, percentile=percentile, threshold_sigma=threshold_sigma, verbose=False, ) raw_lc = tpf.to_lightcurve(method="aperture", aperture_mask=self.aper_mask) # remove nans idx = (np.isnan(raw_lc.time) | np.isnan(raw_lc.flux) | np.isnan(raw_lc.flux_err)) self.tpf = tpf[~idx] self.raw_lc = raw_lc[~idx] if use_pld: if self.verbose: print("Removing scattered light + applying PLD") pld = lk.TessPLDCorrector(self.tpf, aperture_mask=self.aper_mask) if background_mask is None: background_mask = ~self.aper_mask corrected_lc = pld.correct( pixel_components=pixel_components, spline_n_knots=spline_n_knots, spline_degree=spline_degree, background_mask=background_mask, ) self.corrector = pld else: if self.verbose: print("Removing scattered light") # Make a design matrix and pass it to a linear regression corrector regressors = tpf.flux[~idx][:, ~self.aper_mask] dm = (lk.DesignMatrix( regressors, name="pixels").pca(pca_nterms).append_constant()) # Regression Corrector Object rc = lk.RegressionCorrector(self.raw_lc) self.corrector = rc corrected_lc = rc.correct(dm) # Optional: Remove the scattered light, allowing for the large offset from scattered light if with_offset: corrected_lc = (self.raw_lc - rc.model_lc + np.percentile(rc.model_lc.flux, q=5)) lc = corrected_lc.normalize() self.lc_custom = lc # compute Contamination if self.gaia_sources is None: gaia_sources = self.query_gaia_dr2_catalog(radius=120, verbose=False) else: gaia_sources = self.gaia_sources fluxes = get_fluxes_within_mask(self.tpf, self.aper_mask, gaia_sources) self.contratio = sum(fluxes) - 1 if self.tic_params is None: _ = self.query_tic_catalog(return_nearest_xmatch=True) tic_contratio = self.tic_params.contratio dcontratio = abs(tic_contratio - self.contratio) if (tic_contratio is not None) & (dcontratio > 0.5): print(f"contratio: {self.contratio:.2f} (TIC={tic_contratio:.2f})") # add method lc.detrend = lambda: detrend(lc) return lc
def make_custom_lc( self, sector=None, tpf_size=None, sap_mask=None, aper_radius=None, percentile=None, threshold_sigma=None, use_pld=True, pixel_components=3, spline_n_knots=100, spline_degree=3, background_mask=None, pca_nterms=5, with_offset=True, ): """ create a custom lightcurve based on this tutorial: https://docs.lightkurve.org/tutorials/04-how-to-remove-tess-scattered-light-using-regressioncorrector.html Parameters ---------- sector : int or str specific sector or all cutout_size : tuple tpf cutout size aper_radius: int aperture mask radius percentile: float aperture mask percentile threshold_sigma: float aperture mask threshold [sigma] method : float PLD (default) Returns ------- corrected_lc : lightkurve object """ if self.verbose: print("Using lightcurve with custom aperture.") sector = sector if sector is not None else self.sector sap_mask = sap_mask if sap_mask else self.sap_mask aper_radius = aper_radius if aper_radius else self.aper_radius percentile = percentile if percentile else self.percentile threshold_sigma = (threshold_sigma if threshold_sigma else self.threshold_sigma) cutout_size = tpf_size if tpf_size else self.cutout_size tpf_tesscut = self.get_tpf_tesscut(sector=sector, cutout_size=cutout_size) self.aper_mask = parse_aperture_mask( tpf_tesscut, sap_mask=sap_mask, aper_radius=aper_radius, percentile=percentile, threshold_sigma=threshold_sigma, verbose=False, ) raw_lc = tpf_tesscut.to_lightcurve(method="aperture", aperture_mask=self.aper_mask) # remove nans idx = (np.isnan(raw_lc.time) | np.isnan(raw_lc.flux) | np.isnan(raw_lc.flux_err)) self.tpf_tesscut = tpf_tesscut[~idx] self.lc_custom_raw = raw_lc[~idx] if use_pld: if self.verbose: print("Removing scattered light + applying PLD") pld = lk.TessPLDCorrector(self.tpf_tesscut, aperture_mask=self.aper_mask) if background_mask is None: background_mask = ~self.aper_mask corrected_lc = pld.correct( pixel_components=pixel_components, spline_n_knots=spline_n_knots, spline_degree=spline_degree, background_mask=background_mask, ) self.corrector = pld else: if self.verbose: print("Removing scattered light") # Make a design matrix and pass it to a linear regression corrector regressors = tpf_tesscut.flux[~idx][:, ~self.aper_mask] dm = (lk.DesignMatrix( regressors, name="regressors").pca(nterms=pca_nterms).append_constant()) rc = lk.RegressionCorrector(raw_lc) self.corrector = rc corrected_lc = rc.correct(dm) # Optional: Remove the scattered light, allowing for the large offset from scattered light if with_offset: corrected_lc = (raw_lc - rc.model_lc + np.percentile(rc.model_lc.flux, q=5)) lc = corrected_lc.normalize() self.lc_custom = lc # compute Contamination if self.gaia_sources is None: gaia_sources = self.query_gaia_dr2_catalog(radius=120) else: gaia_sources = self.gaia_sources fluxes = get_fluxes_within_mask(self.tpf_tesscut, self.aper_mask, gaia_sources) self.contratio = sum(fluxes) - 1 # add method lc.detrend = lambda: detrend(lc) return lc
def simple_vsr(obs, gradient=False): """Simple estimate of Variable Scan rate Parameters ---------- obs : shadow.Observation Input observation Returns ------- m_spat : np.ndarray Array of the mean variable scan rate g_spat : np.ndarray Array of the gradient of the variable scan rate """ #Cut out dat = np.copy( (obs.sci / obs.flat))[:, obs.spatial.reshape(-1)][:, :, obs.spectral.reshape(-1)] err = np.copy( (obs.err / obs.flat))[:, obs.spatial.reshape(-1)][:, :, obs.spectral.reshape(-1)] # Divide through by average spectrum avg = np.atleast_3d(np.average(dat, weights=1 / err, axis=1)).transpose([0, 2, 1]) dat /= avg err /= avg # ff = np.average(dat[~obs.in_transit], weights=(1/err)[~obs.in_transit], axis=0) # ff = np.atleast_3d(ff).transpose([2, 0, 1]) * np.ones(obs.data.shape) # ff[obs.cosmic_rays] = 1 # ff[obs.error/obs.data > 0.1] = 1 # dat /= ff # err /= ff x = np.linspace(0.5, -0.5, dat.shape[1]) x_p = np.linspace(-0.5, 0.5, dat.shape[1] * 10) # Mean spatial model m_spat = np.zeros((dat.shape[0], dat.shape[1])) if gradient: # Gradient of spatial model g_spat = np.zeros((dat.shape[0], dat.shape[1])) for idx in tqdm(range(dat.shape[0]), desc='Building Simple VSR'): l = lk.LightCurve(x, np.average(dat[idx], weights=1 / err[idx], axis=1)).normalize() m_spat[idx, :] = l.flux if gradient: r = lk.RegressionCorrector(l) dm = lk.designmatrix.create_sparse_spline_matrix( x, knots=list( np.linspace(-0.5, 0.5, int(dat.shape[1] / 2))[1:-1])).append_constant() dm_p = lk.designmatrix.create_sparse_spline_matrix( x_p, knots=list( np.linspace(-0.5, 0.5, int(dat.shape[1] / 2))[1:-1])).append_constant() _ = r.correct(dm, sigma=1e10) model = dm_p.X.dot(r.coefficients) g_spat[idx, :] = np.gradient(model, x_p)[::10] m_spat = np.atleast_3d(m_spat) * np.ones(dat.shape) #g_spat = np.atleast_3d(g_spat) * np.ones(dat.shape) if gradient: return m_spat, g_spat return m_spat
def get_LCs(CLUSTERS,phot_type): NUMBER_PCA_COMPONENTS = 5 #no of vectors for PCA #initialize the masks star_mask=np.empty([len(tpfs),cutout_size,cutout_size],dtype='bool') if phot_type == 'pixel': # if pixel photometry is requested, define a different variable radius_mask=np.empty([len(tpfs),cutout_size,cutout_size],dtype='bool') sky_mask=np.empty([len(tpfs),cutout_size,cutout_size],dtype='bool') frame = tpfs[0].shape[0]//2 # obtain the aperture and the background masks with help of cluster radius and percentile (85%) if phot_type == 'pixel': radius_mask[0],sky_mask[0] = circle_aperture(tpfs[0][frame].flux,tpfs[0][frame].flux, \ Complete_Clusters[Complete_Clusters['NAME'] == CLUSTERS[0]]['CORE_RADIUS'].values[0],85) elif phot_type == 'ensemble': star_mask[0],sky_mask[0] = circle_aperture(tpfs[0][frame].flux,tpfs[0][frame].flux, \ Complete_Clusters[Complete_Clusters['NAME'] == CLUSTERS[0]]['CORE_RADIUS'].values[0],85) # if pixel photometry, iterate over each pixel in the radius mask and download LCs if phot_type == 'pixel': for i in range(len(np.where(radius_mask[0])[0])): star_mask=np.zeros([len(tpfs),cutout_size,cutout_size],dtype='bool') pixel_loc=np.where(radius_mask[0])[0][i],np.where(radius_mask[0])[1][i] star_mask[0][pixel_loc]=True lightcurves = [tpfs[i].to_lightcurve(aperture_mask=star_mask[i]) for i in range(len(tpfs))]; for i in range(len(lightcurves)): lightcurves[i]=lightcurves[i][lightcurves[i].flux_err > 0] #PCA correction regressors = [tpfs[i].flux[:, sky_mask[i]] for i in range(len(tpfs))]# The regressor is the inverse of the aperture #Design a matrix for linear regression dms = [lk.DesignMatrix(r, name='regressors').pca(NUMBER_PCA_COMPONENTS).append_constant() for r in regressors] #Remove noise using linear regression against a DesignMatrix. correctors = [lk.RegressionCorrector(lc) for lc in lightcurves] correctedLCs = [correctors[i].correct(dms[i]) for i in range(len(correctors))]#this is how to subtract the background noise from the star pixels #polyomial correction. to remove long period systematic trends that survived the PCA correction correctedLCs_poly,poly_terms = polynomial_correction(correctedLCs[0]) #save the pixel corrected LCs correctedLCs_poly.to_csv(CLUSTERS[0]+' pixels/core_pixels/corrected_LCs_CORE_RADIUS_pixel_no.{0}'.format(pixel_loc)) # is ensemble photometry, download LC for entire aperture elif phot_type == 'ensemble': lightcurves = [tpfs[i].to_lightcurve(aperture_mask=star_mask[i]) for i in range(len(tpfs))]; for i in range(len(lightcurves)): lightcurves[i]=lightcurves[i][lightcurves[i].flux_err > 0] regressors = [tpfs[i].flux[:, sky_mask[i]] for i in range(len(tpfs))]# The regressor is the inverse of the aperture #Design a matrix for linear regression dms = [lk.DesignMatrix(r, name='regressors').pca(NUMBER_PCA_COMPONENTS).append_constant() for r in regressors] #Remove noise using linear regression against a DesignMatrix. correctors = [lk.RegressionCorrector(lc) for lc in lightcurves] correctedLCs = [correctors[i].correct(dms[i]) for i in range(len(correctors))] #polynomial correction correctedLCs_poly,poly_terms = polynomial_correction(correctedLCs[0]) #calculate the lomb-scargle periodogram for the ensemble corrected LC for a range frequencies t = correctedLCs_poly[correctedLCs_poly.quality == 0].time dy = correctedLCs_poly[correctedLCs_poly.quality == 0].flux_err y = correctedLCs_poly[correctedLCs_poly.quality == 0].flux omega=np.arange(0.05,11,0.01) P_LS = LombScargle(t, y, dy=dy).power(omega)# LS power # max_power=np.max(P_LS) # freq_at_max_power=omega[np.argmax(P_LS)] return omega, P_LS #return frequencies and LS power
def extract_light_curve(fits_filename, outputdir, return_msg=True): # Output name output = Path(fits_filename.stem + '_corrected.pickled') output = outputdir / output # Parameters and Criteria: sigma_clipping = 5 # To be applied after the detrending of the light curve # Structure the data to be saved results = { 'tic': None, 'sector': None, 'ra': None, 'dec': None, 'headers': None, # Headers from the original FITS file 'fit': None, # Result from fit 'neighbours_all': None, # All neighbours stars info 'neighbours_used': None, # Used neighbours stars info 'target': None, # Target star info 'aperture_threshold': None, # HDU to store tabular information 'pca_all': None, # PCA results 'pca_used': None, # PCA results 'centroids': None, # Centroids results 'excluded_intervals': None, # Excluded intervals in days 'lc_raw': None, # Light curves 'lc_raw_nonan': None, # Light curves 'lc_trend': None, # Light curves 'lc_regressed': None, # Light curves 'lc_regressed_notoutlier': None, # Light curves 'median_image': None, 'masks': None, 'tag': None } # Save information from header from original FITS file HDUL, ext = [], 0 while True: try: HDUL.append( fits.getheader(fits_filename.as_posix(), ext=ext).tostring()) ext += 1 except IndexError: break except Exception as e: print('Unexpected exception when reading headers from FITS: ', e) break results['headers'] = HDUL # Load the TESS taret pixel file try: tpf = lk.TessTargetPixelFile(fits_filename.as_posix()) except Exception as e: # Save results err_msg = f'"lightkurve.TessTargetPixelFile()" could not open file {fits_filename.as_posix()}. Exception: {e}' results['tag'] = err_msg picklefile = open(output.as_posix(), 'wb') pickle.dump(results, picklefile) picklefile.close() if return_msg: return err_msg return tic = str(tpf.get_keyword('ticid')) sector = str(tpf.get_keyword('sector')) target_ra = np.float(tpf.ra) target_dec = np.float(tpf.dec) # Store to results results['tic'], results['sector'] = tic, sector results['ra'], results['dec'] = target_ra, target_dec # Initialize messages id_msg = f'TIC {tic} Sector {sector}: Skipped: ' OK_msg = f'TIC {tic} Sector {sector}: OK' # Calculate the median image with warnings.catch_warnings(): warnings.simplefilter('ignore') median_image = np.nanmedian(tpf.flux, axis=0) # Store to results results['median_image'] = median_image # Estimate of aperture mask and background mask ap_mask_threshold, bkg_mask_threshold = 5, 3 ap_mask = threshold_mask(median_image, threshold=ap_mask_threshold, reference_pixel='center') ap_bkg = ~threshold_mask( median_image, threshold=bkg_mask_threshold, reference_pixel=None) # Exclude NaN values outside the camera ap_bkg &= ~np.isnan(median_image) # Estimate the median flux background median_bkg_flux = np.median(median_image[ap_bkg]) # Store to results results['masks'] = {'aperture':ap_mask,\ 'background':ap_bkg} # Check validity of aperture mask OK_ap_mask, err_msg = check_aperture_mask(ap_mask, id_msg) # If aperture is not good, exit program with corresponding message if not OK_ap_mask: # Save results results['tag'] = err_msg picklefile = open(output.as_posix(), 'wb') pickle.dump(results, picklefile) picklefile.close() if return_msg: return err_msg return # Refine aperture try: WCS = tpf.wcs except IndexError: # Save results err_msg = id_msg + 'No WCS info in header' results['tag'] = err_msg picklefile = open(output.as_posix(), 'wb') pickle.dump(results, picklefile) picklefile.close() if return_msg: return err_msg return ap_mask,\ target_coord_pixel, target_tmag,\ nb_coords_pixel, nb_tmags,\ err_msg = refine_aperture(results, tic, target_ra, target_dec, WCS,\ ap_mask, ap_mask_threshold, median_image, prepend_err_msg=id_msg) # If not satisfactory aperture mask if ap_mask is None: # Save results results['tag'] = err_msg picklefile = open(output.as_posix(), 'wb') pickle.dump(results, picklefile) picklefile.close() if return_msg: return err_msg return # Variation in time of aperture's center of mass centroid_col, centroid_row = tpf.estimate_centroids(aperture_mask=ap_mask, method='quadratic') centroid_col -= tpf.column centroid_row -= tpf.row sqrt_col2_row2 = np.sqrt(centroid_col**2 + centroid_row**2) # Store to results results['centroids'] = {'col':centroid_col,\ 'row':centroid_row,\ 'sqrt_col2_row2':sqrt_col2_row2,\ 'time':tpf.time} # Fit the image and find the contamination fraction within the aperture mask fitted_image, err_msg = contamination(results, median_image,ap_mask,\ target_coord_pixel, target_tmag,\ nb_coords_pixel, nb_tmags,\ tpf.wcs,median_bkg_flux, prepend_err_msg=id_msg) if fitted_image is None: # Save results results['tag'] = err_msg picklefile = open(output.as_posix(), 'wb') pickle.dump(results, picklefile) picklefile.close() if return_msg: return err_msg return # Generate the raw light curve lc_raw = tpf.to_lightcurve(aperture_mask=ap_mask, method='aperture') # Store to results results['lc_raw'] = {'flux':lc_raw.flux,\ 'time':lc_raw.time} # Find the indices of the quality mask that created the light curve ind = np.argwhere(tpf.quality_mask == True) # Masks with True value the light curve times with null or NaN flux mask = lc_raw.flux == 0 mask |= lc_raw.flux == np.nan # Set to False the indices to be masked (ignored). # Note that we take a subset from `ind` because the masks where defined from the light curve tpf.quality_mask[ind[mask]] = False # Exclude intervals previously decided exclude_interval(tpf, sector, results) # Generate the light curve lc = tpf.to_lightcurve(aperture_mask=ap_mask, method='aperture') # Store to results results['lc_raw_nonan'] = {'flux':lc.flux,\ 'time':lc.time} # Make a design matrix and pass it to a linear regression corrector regressors = tpf.flux[:, ap_bkg] # Number of PCs to use npc, dm, rc = find_number_of_PCs(results, regressors, lc) if npc == 0: # Save results results['tag'] = id_msg + 'None PC used, no detrended done.' picklefile = open(output.as_posix(), 'wb') pickle.dump(results, picklefile) picklefile.close() if return_msg: return err_msg return try: # Detrend light curve using PCA dm = lk.DesignMatrix(regressors, name='regressors').pca(npc).append_constant() rc = lk.RegressionCorrector(lc) lc_regressed = rc.correct(dm) lc_trend = rc.diagnostic_lightcurves['regressors'] # Sigma clipping the remove outliers lc_regressed_no_outliers, lc_mask_regressed_outliers = lc_regressed.remove_outliers( return_mask=True, sigma=sigma_clipping) # Store to results results['lc_trend'] = {'flux':lc_trend.flux,\ 'time':lc_trend.time} results['lc_regressed'] = {'flux':lc_regressed.flux,\ 'time':lc_regressed.time,\ 'outlier_mask':lc_mask_regressed_outliers,\ 'sigma_clipping':sigma_clipping} results['lc_regressed_notoutlier'] = {'flux':lc_regressed_no_outliers.flux,\ 'time':lc_regressed_no_outliers.time} results['pca_used'] = {'coef':rc.coefficients,\ 'pc':[dm.values[:,i] for i in range(dm.rank)],\ 'dm':dm,\ 'rc':rc,\ 'npc':npc} # Save results results['tag'] = 'OK' picklefile = open(output.as_posix(), 'wb') pickle.dump(results, picklefile) picklefile.close() if return_msg: return OK_msg return except Exception as e: print('!!!!!!!!!!!!!!!!!!!!!!!!!') print(f' Sector {sector}.') print('EXCEPTION:', e) print('!!!!!!!!!!!!!!!!!!!!!!!!!') if return_msg: return id_msg + '::' + repr(e) + '::' + str(e) return
lkf = lkf[bidx] color_print('\nAperture chosen: ', 'lightcyan', str(bidx+1) + 'px radius' if args.circ==0 else 'No. ' + str(bidx), 'default', '\tNumber of pixels inside: ', 'lightcyan', str(dap[bidx].sum()), 'default') #PCA if args.pca: import lightkurve regressors = flux[:, ~dap[bidx]] dm = lightkurve.DesignMatrix(regressors, name='regressors') dm = dm.pca(5) dm = dm.append_constant() corrector = lightkurve.RegressionCorrector(lkf) corr_flux = corrector.correct(dm) lkf = lkf - corrector.model_lc + np.percentile(corrector.model_lc.flux, 5) #lkf.flux = lkf.flux - corrector.model_lc + np.percentile(corrector.model_lc.flux, 5) #PLD? elif args.pld: ''' if iP is None: mask = np.ones(time.size).astype(bool) else: mask = mask_planet(time, it0, iP, dur=idur) flux_pld -= bkgs[:,None,None] pldflsum = np.nansum(flux_pld, axis=0) pldthr = mad_std(pldflsum)
def calculate_contamination( tpfs, period, t0, duration, sigma=5, plot=True, cbvs=True, sff=False, windows=20, bins=5, spline_period=1, **kwargs, ): """Calculate the contamination for a target Parameters ---------- period : float Period of transiting object in days t0 : float Transit midpoint of transiting object in days duration : float Duration of transit in days sigma : float The significance level at which to create an aperture for the contaminanting source. If the apertures are large, try increasing sigma. If the apertures are small, or contaminante fails, you could try (slightly) lowering sigma. plot: bool If True, will generate a figure cbvs : bool If True, will use Kepler/TESS CBVs to detrend. Default is True sff : bool If True, will use the SFF method to correct variability. Default is False. spline_period : float The period of a spline to fit. For short period variability, set this value to a smaller number. Default is 0.75 days. Returns ------- result : list of dict List of dictionaries containing the contamination properties If plot is True, will show a figure, and will put the matplotlib.pyplot.figure object into the result dictionary. """ if isinstance(tpfs, (list)): tpfs = TPFC(tpfs) elif isinstance(tpfs, TPF): tpfs = TPFC([tpfs]) elif not isinstance(tpfs, TPFC): raise ValueError( "please pass `lk.TargetPixelFile` or `lk.TargetPixelFileCollection`" ) # remove nans for idx, tpf in enumerate(tpfs): aper = tpf.pipeline_mask if not (aper.any()): aper = tpf.create_threshold_mask() mask = (np.abs( (tpf.pos_corr1)) < 5) & ((np.gradient(tpf.pos_corr2)) < 5) mask &= np.nan_to_num(tpf.to_lightcurve(aperture_mask=aper).flux) != 0 if tpf.mission.lower() in ["k2", "ktwo"]: # For k2 we have to get rid of some serious outliers... r, c = tpf.estimate_centroids("all") r, c = np.nan_to_num(r.value), np.nan_to_num(c.value) arclength = np.hypot(r - np.min(r), c - np.min(c)) reg = lk.RegressionCorrector( lk.LightCurve(time=tpf.time, flux=np.gradient(arclength))) dm = lk.correctors.designmatrix.create_sparse_spline_matrix( tpf.time.value, n_knots=int((tpf.time[-1].value - tpf.time[0].value) // 1), ).append_constant() _ = reg.correct(dm, sigma=3) # If there aren't too many outliers if (reg.outlier_mask.sum()) / len(reg.outlier_mask) < 0.2: mask &= ~reg.outlier_mask tpfs[idx] = tpfs[idx][mask] results = [] for tpf in tqdm(tpfs, desc="Modeling TPFs"): aper = tpf.pipeline_mask if not (aper.any()): aper = tpf.create_threshold_mask() lc = tpf.to_lightcurve(aperture_mask=aper).normalize() if sff: with warnings.catch_warnings(): warnings.simplefilter("ignore") s = lk.correctors.SFFCorrector(lc) _ = s.correct(windows=windows, bins=bins, timescale=spline_period) sff_dm = s.dmc bls = lc.to_periodogram("bls", period=[period, period], duration=duration) t_mask = bls.get_transit_mask(period=period, transit_time=t0, duration=duration) # Correct light curve if cbvs: with warnings.catch_warnings(): warnings.simplefilter("ignore") cbv_array = (lk.correctors.CBVCorrector( tpf.to_lightcurve(aperture_mask=aper), interpolate_cbvs=True, extrapolate_cbvs=True, ).cbvs[0].to_designmatrix().X[:, :4]) else: cbv_array = None r1, c1 = tpf.estimate_centroids(aperture_mask=aper) r1 -= np.median(r1) c1 -= np.median(c1) X = build_X( lc.time.jd, r1.value, c1.value, cbvs=cbv_array, windows=windows, bins=bins, spline_period=spline_period, sff=sff, **kwargs, ) X = X[:, np.asarray(X.sum(axis=0))[0] != 0] with warnings.catch_warnings(): warnings.simplefilter("ignore") dm1 = lk.SparseDesignMatrix( X, name="X", prior_mu=np.hstack([np.zeros(X.shape[1] - 1), 0]), prior_sigma=np.hstack([np.ones(X.shape[1] - 1) * 1e2, 0.1]), ) r = lk.RegressionCorrector(lc.copy()) target = r.correct(dm1, cadence_mask=~t_mask) stellar_lc = r.diagnostic_lightcurves["X"].flux if t_mask.sum() == 0: results.append( _package_results( tpf, target=target, contaminator=target * 0, aper=aper, contaminant_aper=aper * False, transit_pixels=np.zeros(aper.shape), transit_pixels_err=np.inf * np.ones(aper.shape), period=period, t0=t0, duration=duration, plot=plot, )) continue # Find a transit model bls = target.to_periodogram("bls", period=[period, period], duration=duration) t_model = (~bls.get_transit_mask(period=period, transit_time=t0, duration=duration)).astype(float) - 1 depth = bls.compute_stats(period=period, transit_time=t0, duration=duration)["depth"][0] X = build_X( lc.time.jd, r1.value, c1.value, flux=stellar_lc, t_model=t_model, cbvs=cbv_array, windows=windows, bins=bins, spline_period=spline_period, sff=sff, **kwargs, ) X = X[:, np.asarray(X.sum(axis=0))[0] != 0] with warnings.catch_warnings(): warnings.simplefilter("ignore") dm = lk.SparseDesignMatrix( X, name="X", prior_mu=np.hstack([np.zeros(X.shape[1] - 1), depth]), prior_sigma=np.hstack([np.ones(X.shape[1] - 1) * 1e4, 0.1]), ) dm_no_transit = lk.SparseDesignMatrix( X[:, :-1], name="X", prior_mu=np.hstack([np.zeros(X.shape[1] - 1)]), prior_sigma=np.hstack([np.ones(X.shape[1] - 1) * 1e4]), ) model = np.zeros(tpf.flux.shape) model_err = np.zeros(tpf.flux.shape) # Hard coded saturation limit. Probably not ideal. saturated = np.percentile(np.nan_to_num(tpf.flux.value), 95, axis=0) > 1.5e5 if saturated.any(): dsat = np.gradient(saturated.astype(float), axis=0) if (~np.any(dsat == 0.5)) | (~np.any(dsat == -0.5)): raise ValueError( "Too close to a saturation column that isn't fully captured." ) saturated |= np.abs(dsat) != 0 pixels = tpf.flux.value.copy() pixels_err = tpf.flux_err.value.copy() transit_pixels = np.zeros(tpf.flux.shape[1:]) transit_pixels_err = np.zeros(tpf.flux.shape[1:]) * np.inf chi_ratio = np.zeros(tpf.flux.shape[1:]) * np.inf for jdx, s in enumerate(saturated.T): if any(s): l = np.where(s)[0][s.sum() // 2] pixels[:, s, jdx] = np.nan pixels[:, l, jdx] = tpf.flux.value[:, s, jdx].sum(axis=(1)) pixels_err[:, l, jdx] = ((tpf.flux_err.value[:, s, jdx]** 2).sum(axis=(1))**0.5) / s.sum() for idx in range(tpf.shape[1]): for jdx in range(tpf.shape[2]): if np.nansum(pixels[:, idx, jdx]) == 0: continue # If we wanted a box smooth... # lb1, lb2 = np.max([idx - 1, 0]), np.max([jdx - 1, 0]) # ub1, ub2 = np.min([idx + 2, pixels.shape[1]]), np.min( # [jdx + 2, pixels.shape[2]] # ) # # r.lc.flux = np.nansum( # pixels[:, lb1:ub1][:, :, lb2:ub2], # axis=(1, 2), # ) # r.lc.flux_err = ( # np.nansum((pixels_err[:, lb1:ub1][:, :, lb2:ub2] ** 2), axis=(1, 2)) # ** 0.5 # ) # r.lc.flux_err /= np.nanmedian(r.lc.flux) # r.lc.flux /= np.nanmedian(r.lc.flux) # clc = r.correct(dm) r.lc.flux = pixels[:, idx, jdx] / np.median(pixels[:, idx, jdx]) r.lc.flux_err = pixels_err[:, idx, jdx] / np.median( pixels[:, idx, jdx]) if sff: # Do a first multiplicative correction _ = r.correct(sff_dm, cadence_mask=~t_mask) mlc = r.model_lc + np.median(r.lc.flux) mlc /= np.percentile(mlc.flux, 95) r.lc /= mlc r.lc = r.lc / np.median(r.lc.flux) clc = r.correct(dm) transit_pixels[idx, jdx] = r.coefficients[-1] sigma_w_inv = X.T.dot(X / r.lc.flux_err[:, None]**2) + np.diag( 1 / dm.prior_sigma**2) transit_pixels_err[idx, jdx] = (np.asarray( np.linalg.inv(sigma_w_inv)).diagonal()[-1]**0.5) for jdx, s in enumerate(saturated.T): if any(s): l = np.where(s)[0][s.sum() // 2] transit_pixels[s, jdx] = transit_pixels[l, jdx] transit_pixels_err[s, jdx] = transit_pixels_err[l, jdx] with warnings.catch_warnings(): warnings.simplefilter("ignore") contaminant_aper = create_threshold_mask( transit_pixels / transit_pixels_err, sigma) if tpf.mission.lower() in ["k2", "ktwo"]: # Ktwo is noisy so we have to make the aperture a bit bigger. contaminant_aper |= (np.asarray( np.gradient(contaminant_aper.astype(float))) != 0).any(axis=0) contaminated_lc = tpf.to_lightcurve( aperture_mask=contaminant_aper).normalize() r.lc = contaminated_lc contaminator = r.correct(dm1, cadence_mask=~t_mask) results.append( _package_results( tpf, target=target, contaminator=contaminator, aper=aper, contaminant_aper=contaminant_aper, transit_pixels=transit_pixels, transit_pixels_err=transit_pixels_err, period=period, t0=t0, duration=duration, plot=plot, )) return results
def __init__(self, targetpixelfile, gaia=True, magnitude_limit=18, frequencies=[], frequnit=u.uHz, principle_components=5, aperture=None, **kwargs): #Defining an aperture that will be used in plotting and making empty 2-d arrays of the correct size for masks if targetpixelfile.pipeline_mask.any() == False: self.aperture = aperture else: self.aperture = targetpixelfile.pipeline_mask self.tpf = targetpixelfile # Make a design matrix and pass it to a linear regression corrector self.raw_lc = self.tpf.to_lightcurve(aperture_mask=self.aperture) self.dm = lk.DesignMatrix( self.tpf.flux[:, ~self.tpf.create_threshold_mask()], name='regressors').pca(principle_components) rc = lk.RegressionCorrector(self.raw_lc) corrected_lc = rc.correct(self.dm.append_constant()) corrected_lc[np.where(corrected_lc.quality == 0)] self.corrected_lc = corrected_lc.remove_outliers() self.frequency_list = np.asarray((frequencies * frequnit).to(1 / u.d)) self.principle_components = principle_components def Obtain_Initial_Phase(tpf, corrected_lc, frequency_list): flux = corrected_lc.flux.value times = corrected_lc.time.value - np.mean(corrected_lc.time.value) pg = corrected_lc.to_periodogram(frequency=np.append( [0.0001], frequency_list), ls_method='slow') initial_flux = np.asarray(pg.power[1:]) initial_phase = np.zeros(len(frequency_list)) def lc_model(time, amp, freq, phase): return amp * np.sin(2 * np.pi * freq * time + phase) def background_model(time, height): return np.ones(len(time)) * height for j in np.arange(len(frequency_list)): for i in np.arange(len(frequency_list)): if (i == 0): model = lm.Model(lc_model, independent_vars=['time'], prefix='f{0:d}'.format(i)) model += lm.Model(background_model, independent_vars=['time']) else: model += lm.Model(lc_model, independent_vars=['time'], prefix='f{0:d}'.format(i)) model.set_param_hint('f{0:d}phase'.format(i), min=-np.pi, max=np.pi, value=initial_phase[i], vary=False) model.set_param_hint('f{0:d}amp'.format(i), value=initial_flux[i], vary=False) model.set_param_hint('height', value=np.mean(flux), vary=False) model.set_param_hint('f{0:d}freq'.format(i), value=frequency_list[i], vary=False) params = model.make_params() params['f{0:d}phase'.format(j)].set(vary=True) params['f{0:d}phase'.format(j)].set(value=initial_phase[j]) params['f{0:d}phase'.format(j)].set(brute_step=np.pi / 10) result = model.fit(corrected_lc.flux.value, params, time=times, method='brute') initial_phase[j] = result.best_values['f{0:d}phase'.format(j)] return initial_phase self.initial_phases = Obtain_Initial_Phase(self.tpf, self.corrected_lc, self.frequency_list) def Obtain_Final_Phase(tpf, corrected_lc, frequency_list, initial_phases): flux = corrected_lc.flux.value times = corrected_lc.time.value - np.mean(corrected_lc.time.value) pg = corrected_lc.to_periodogram(frequency=np.append( [0.0001], frequency_list), ls_method='slow') initial_flux = np.asarray(pg.power[1:]) def lc_model(time, amp, freq, phase): return amp * np.sin(2 * np.pi * freq * time + phase) def background_model(time, height): return np.ones(len(time)) * height for i in np.arange(len(frequency_list)): if (i == 0): model = lm.Model(lc_model, independent_vars=['time'], prefix='f{0:d}'.format(i)) model += lm.Model(background_model, independent_vars=['time']) else: model += lm.Model(lc_model, independent_vars=['time'], prefix='f{0:d}'.format(i)) model.set_param_hint('f{0:d}phase'.format(i), min=-np.pi, max=np.pi, value=initial_phases[i], vary=True) model.set_param_hint('f{0:d}amp'.format(i), value=initial_flux[i], vary=True) model.set_param_hint('height', value=np.mean(flux), vary=True) model.set_param_hint('f{0:d}freq'.format(i), value=frequency_list[i], vary=False) params = model.make_params() result = model.fit(corrected_lc.flux.value, params, time=times) final_phases = [ result.best_values['f{0:d}phase'.format(j)] for j in np.arange(len(frequency_list)) ] return final_phases self.final_phases = Obtain_Final_Phase(self.tpf, self.corrected_lc, self.frequency_list, self.initial_phases) def Obtain_Final_Fit(tpf, corrected_lc, frequency_list, final_phases): flux = corrected_lc.flux.value times = corrected_lc.time.value - np.mean(corrected_lc.time.value) pg = corrected_lc.to_periodogram(frequency=np.append( [0.0001], frequency_list), ls_method='slow') initial_flux = np.asarray(pg.power[1:]) def lc_model(time, amp, freq, phase): return amp * np.sin(2 * np.pi * freq * time + phase) def background_model(time, height): return np.ones(len(time)) * height for i in np.arange(len(frequency_list)): if (i == 0): model = lm.Model(lc_model, independent_vars=['time'], prefix='f{0:d}'.format(i)) model += lm.Model(background_model, independent_vars=['time']) else: model += lm.Model(lc_model, independent_vars=['time'], prefix='f{0:d}'.format(i)) model.set_param_hint('f{0:d}phase'.format(i), value=final_phases[i], vary=False) model.set_param_hint('f{0:d}amp'.format(i), value=initial_flux[i], vary=True) model.set_param_hint('height', value=np.mean(flux), vary=True) model.set_param_hint('f{0:d}freq'.format(i), value=frequency_list[i], vary=False) params = model.make_params() result = model.fit(corrected_lc.flux.value, params, time=times) return result heats = [] heats_error = [] #Iterating through columns of pixels for i in np.arange(0, len(self.aperture)): #Iterating through rows of pixels for j in np.arange(0, len(self.aperture[0])): #Making an empty 2-d array mask = np.zeros((len(self.aperture), len(self.aperture[0])), dtype=bool) #Iterating to isolate pixel by pixel to get light curves mask[i][j] = True #Getting the light curve for a pixel and excluding any flagged data lightcurve = self.tpf.to_lightcurve(aperture_mask=mask) rcc = lk.RegressionCorrector(lightcurve) lc = rcc.correct(self.dm.append_constant()) #lc = lc[np.where(lc.quality == 0)] #lc = lc.remove_outliers() bestfit = Obtain_Final_Fit(self.tpf, lc, self.frequency_list, self.final_phases) heat = np.asarray([ bestfit.best_values['f{0:d}amp'.format(n)] for n in np.arange(len(self.frequency_list)) ]) #heat = bestfit.best_values['f0amp']# / bestfit.params['f0amp'].stderr heat_error = np.asarray([ bestfit.params['f{0:d}amp'.format(n)].stderr for n in np.arange(len(self.frequency_list)) ]) #Extending the list of fitting data for each pixel heats.extend([heat]) heats_error.extend([heat_error]) #Taking the final list and turning it into a 2-d numpy array with the same dimensions of the full postage stamp #heats = np.reshape(np.asarray(heats),(len(self.aperture),len(self.aperture[0]))) #heats_error = np.reshape(np.asarray(heats_error),(len(self.aperture),len(self.aperture[0]))) heats = np.asarray(heats) heats_error = np.asarray(heats_error) #Defining self.periodogram as this 2-d array of periodogram data self.heatmap = heats.T self.heatmap_error = heats_error.T self.timeserieslength = (self.tpf.time.max() - self.tpf.time.min()).value self.gaiadata = None if (gaia == True): """Make the Gaia Figure Elements""" # Get the positions of the Gaia sources c1 = SkyCoord(self.tpf.ra, self.tpf.dec, frame='icrs', unit='deg') # Use pixel scale for query size pix_scale = 4.0 # arcseconds / pixel for Kepler, default if self.tpf.mission == 'TESS': pix_scale = 21.0 # We are querying with a diameter as the radius, overfilling by 2x. from astroquery.vizier import Vizier Vizier.ROW_LIMIT = -1 result = Vizier.query_region( c1, catalog=["I/345/gaia2"], radius=Angle(np.max(self.tpf.shape[1:]) * pix_scale, "arcsec")) no_targets_found_message = ValueError( 'Either no sources were found in the query region ' 'or Vizier is unavailable') too_few_found_message = ValueError( 'No sources found brighter than {:0.1f}'.format( magnitude_limit)) if result is None: raise no_targets_found_message elif len(result) == 0: raise too_few_found_message result = result["I/345/gaia2"].to_pandas() result = result[result.Gmag < magnitude_limit] if len(result) == 0: raise no_targets_found_message year = ((self.tpf.time[0].jd - 2457206.375) * u.day).to(u.year) pmra = ((np.nan_to_num(np.asarray(result.pmRA)) * u.milliarcsecond / u.year) * year).to(u.deg).value pmdec = ((np.nan_to_num(np.asarray(result.pmDE)) * u.milliarcsecond / u.year) * year).to(u.deg).value result.RA_ICRS += pmra result.DE_ICRS += pmdec radecs = np.vstack([result['RA_ICRS'], result['DE_ICRS']]).T coords = self.tpf.wcs.all_world2pix(radecs, 0) # Gently size the points by their Gaia magnitude sizes = 64.0 / 2**(result['Gmag'] / 5.0) one_over_parallax = 1.0 / (result['Plx'] / 1000.) source = dict(ra=result['RA_ICRS'], dec=result['DE_ICRS'], source=result['Source'].astype(str), Gmag=result['Gmag'], plx=result['Plx'], one_over_plx=one_over_parallax, x=coords[:, 0], y=coords[:, 1], size=sizes) self.gaiadata = source class frequency_heatmap: def __init__(self, tpf, heats, heats_error, frequencies, gaia_data): self.heat_stamp = heats self.gaiadata = gaia_data self.heatmap_error = heats_error self.size = tpf.pipeline_mask.shape self.frequencies = frequencies def centroid(self): #Residuals to minimize relative to the error bars def residual(params, amp, amperr): x = params['x'] y = params['y'] sigma = params['sigma'] xpix, ypix = np.meshgrid(np.arange(self.size[0]), np.arange(self.size[1])) res = [] for i in np.arange(len(self.frequencies)): height = params['height{0:d}'.format(i)] model = height * np.exp(-(((x - xpix) / sigma)**2 + ((y - ypix) / sigma)**2) / 2) res.extend([(amp[i].reshape(self.size) - model) / amperr[i].reshape(self.size)]) return np.asarray(res) #Set starting values to converge from self.heatmap_error[np.where( self.heatmap_error == None)] = np.nan composite_heatmap = self.heat_stamp.sum(axis=0).reshape( self.size) / ((np.nansum( self.heatmap_error**2, axis=0))**(1 / 2)).reshape( self.size) #issue with numpy using sqrt? c = np.where(composite_heatmap == composite_heatmap.max()) params = Parameters() for i in np.arange(len(frequencies)): params.add('height{0:d}'.format(i), value=np.max(self.heat_stamp[i])) params.add('x', value=c[1][0]) params.add('y', value=c[0][0]) params.add('sigma', value=1) #Do the fit minner = Minimizer(residual, params, fcn_args=(self.heat_stamp, self.heatmap_error)) self.result = minner.minimize() fit = self.result.params.valuesdict() self.x = fit['x'] self.y = fit['y'] def star_list(self): gaia_data = self.gaiadata no_gaia_data_message = ValueError( 'No gaia data initialized in PixelMapPeriodogram class') if gaia_data == None: raise no_gaia_data_message else: distances = np.square(self.x - gaia_data['x']) + np.square( self.y - gaia_data['y']) closest_star_mask = np.where( np.square(self.x - gaia_data['x']) + np.square(self.y - gaia_data['y']) == ( np.square(self.x - gaia_data['x']) + np.square(self.y - gaia_data['y'])).min()) stars = dict( ra=np.asarray(gaia_data['ra']), dec=np.asarray(gaia_data['dec']), source=np.asarray(gaia_data['source']), x=np.asarray(gaia_data['x']), y=np.asarray(gaia_data['y']), distance=distances, probability=2 * stats.norm.sf( distances, scale=np.sqrt(self.result.params['x'].stderr**2 + self.result.params['y'].stderr**2)) ) #I believe mutiply by 2 since we wont have a negative distance starlist = pd.DataFrame.from_dict(stars) self.stars = starlist.sort_values(by=[r'distance']) fh = frequency_heatmap(self.tpf, self.heatmap, self.heatmap_error, self.frequency_list, self.gaiadata) fh.centroid() fh.star_list() self.centroid = [fh.x, fh.y] self.heatmap = self.heatmap.sum(axis=0).reshape( self.aperture.shape[0], self.aperture.shape[1]) / np.sqrt( (self.heatmap_error**2).sum(axis=0)).reshape( self.aperture.shape[0], self.aperture.shape[1]) self.starfit = fh.stars.reset_index() self.result = fh.result
def build(self, object_info, sherlock_dir): mission_id = object_info.mission_id() sherlock_id = object_info.sherlock_id() quarters = None sectors = None logging.info("Retrieving star catalog info...") mission, mission_prefix, id = super().parse_object_id(mission_id) if mission_prefix not in self.star_catalogs: raise ValueError("Wrong object id " + mission_id) star_info = starinfo.StarInfo( sherlock_id, *self.star_catalogs[mission_prefix].catalog_info(id)) logging.info("Downloading lightcurve files...") sectors = None if object_info.sectors == 'all' or mission != "TESS" else object_info.sectors quarters = None if object_info.sectors == 'all' or mission != "K2" else object_info.sectors campaigns = None if object_info.sectors == 'all' or mission != "Kepler" else object_info.sectors if object_info.aperture_file is None: lcf = lk.search_lightcurve(str(mission_id), mission=mission, cadence="short", sector=sectors, quarter=quarters, campaign=campaigns, author=self.authors[mission])\ .download_all() if lcf is None: raise ObjectProcessingError( "Light curve not found for object id " + mission_id) lc = None matching_objects = [] for i in range(0, len(lcf.PDCSAP_FLUX)): if lcf.PDCSAP_FLUX[i].label == mission_id: if lc is None: lc = lcf.PDCSAP_FLUX[i].normalize() else: lc = lc.append(lcf.PDCSAP_FLUX[i].normalize()) else: matching_objects.append(lcf.PDCSAP_FLUX[i].label) if len(matching_objects) > 0: logging.warning( "================================================") logging.warning("TICS IN THE SAME PIXEL: " + str(matching_objects)) logging.warning( "================================================") lc = lc.remove_nans() transits_min_count = self.__calculate_transits_min_count(len(lcf)) if mission_prefix == self.MISSION_ID_KEPLER or mission_id == self.MISSION_ID_KEPLER_2: quarters = [lcfile.quarter for lcfile in lcf] elif mission_prefix == self.MISSION_ID_TESS: sectors = [file.sector for file in lcf] if mission_prefix == self.MISSION_ID_KEPLER_2: logging.info("Correcting K2 motion in light curve...") quarters = [lcfile.campaign for lcfile in lcf] lc = lc.to_corrector("sff").correct(windows=20) return lc, star_info, transits_min_count, np.unique( sectors), np.unique(quarters) else: logging.info("Using user apertures!") tpfs = lk.search_targetpixelfile( str(mission_id), mission=mission, cadence="short", sector=sectors, quarter=quarters, campaign=campaigns, author=authors[mission]).download_all() apertures = {} if isinstance(object_info.aperture_file, str): aperture = [] with open(object_info.aperture_file, 'r') as fd: reader = csv.reader(fd) for row in reader: aperture.append(row) aperture = np.array(aperture) for tpf in tpfs: if mission_prefix == self.MISSION_ID_KEPLER: apertures[tpf.quarter] = aperture elif mission_prefix == self.MISSION_ID_TESS: apertures[tpf.sector] = aperture elif mission_prefix == self.MISSION_ID_KEPLER_2: apertures[tpf.campaign] = aperture else: for sector, aperture_file in object_info.aperture_file.items(): aperture = [] with open(aperture_file, 'r') as fd: reader = csv.reader(fd) for row in reader: aperture.append(row) aperture = np.array(aperture) apertures[sector] = aperture lc = None for tpf in tpfs: if mission_prefix == self.MISSION_ID_KEPLER: sector = tpf.quarter elif mission_prefix == self.MISSION_ID_TESS: sector = tpf.sector elif mission_prefix == self.MISSION_ID_KEPLER_2: sector = tpf.campaign aperture = apertures[sector].astype(bool) tpf.plot(aperture_mask=aperture, mask_color='red') plt.savefig(sherlock_dir + "/Aperture_[" + str(sector) + "].png") plt.close() if mission_prefix == self.MISSION_ID_KEPLER: corrector = lk.KeplerCBVCorrector(tpf) corrector.plot_cbvs([1, 2, 3, 4, 5, 6, 7]) raw_lc = tpf.to_lightcurve( aperture_mask=aperture).remove_nans() plt.savefig(sherlock_dir + "/Corrector_components[" + str(sector) + "].png") plt.close() it_lc = corrector.correct([1, 2, 3, 4, 5]) ax = raw_lc.plot(color='C3', label='SAP Flux', linestyle='-') it_lc.plot(ax=ax, color='C2', label='CBV Corrected SAP Flux', linestyle='-') plt.savefig(sherlock_dir + "/Raw_vs_CBVcorrected_lc[" + str(sector) + "].png") plt.close() elif mission_prefix == self.MISSION_ID_KEPLER_2: raw_lc = tpf.to_lightcurve( aperture_mask=aperture).remove_nans() it_lc = raw_lc.to_corrector("sff").correct(windows=20) ax = raw_lc.plot(color='C3', label='SAP Flux', linestyle='-') it_lc.plot(ax=ax, color='C2', label='CBV Corrected SAP Flux', linestyle='-') plt.savefig(sherlock_dir + "/Raw_vs_SFFcorrected_lc[" + str(sector) + "].png") plt.close() elif mission_prefix == self.MISSION_ID_TESS: temp_lc = tpf.to_lightcurve(aperture_mask=aperture) where_are_NaNs = np.isnan(temp_lc.flux) temp_lc = temp_lc[np.where(~where_are_NaNs)] regressors = tpf.flux[np.argwhere(~where_are_NaNs), ~aperture] temp_token_lc = [ temp_lc[i:i + 2000] for i in range(0, len(temp_lc), 2000) ] regressors_token = [ regressors[i:i + 2000] for i in range(0, len(regressors), 2000) ] it_lc = None raw_it_lc = None item_index = 0 for temp_token_lc_item in temp_token_lc: regressors_token_item = regressors_token[item_index] design_matrix = lk.DesignMatrix( regressors_token_item, name='regressors').pca(5).append_constant() corr_lc = lk.RegressionCorrector( temp_token_lc_item).correct(design_matrix) if it_lc is None: it_lc = corr_lc raw_it_lc = temp_token_lc_item else: it_lc = it_lc.append(corr_lc) raw_it_lc = raw_it_lc.append(temp_token_lc_item) item_index = item_index + 1 ax = raw_it_lc.plot(label='Raw light curve') it_lc.plot(ax=ax, label='Corrected light curve') plt.savefig(sherlock_dir + "/Raw_vs_DMcorrected_lc[" + str(sector) + "].png") plt.close() if lc is None: lc = it_lc.normalize() else: lc = lc.append(it_lc.normalize()) lc = lc.remove_nans() lc.plot(label="Normalized light curve") plt.savefig(sherlock_dir + "/Normalized_lc[" + str(sector) + "].png") plt.close() transits_min_count = self.__calculate_transits_min_count(len(tpfs)) if mission_prefix == self.MISSION_ID_KEPLER or mission_id == self.MISSION_ID_KEPLER_2: quarters = [lcfile.quarter for lcfile in tpfs] elif mission_prefix == self.MISSION_ID_TESS: sectors = [file.sector for file in tpfs] if mission_prefix == self.MISSION_ID_KEPLER_2: logging.info("Correcting K2 motion in light curve...") quarters = [lcfile.campaign for lcfile in tpfs] return lc, star_info, transits_min_count, np.unique( sectors), np.unique(quarters)