def test_apogee_continuum(self): raw_spectra = np.ones((10, 8575)) * 2 raw_spectra_err = np.zeros((10, 8575)) # continuum cont_spectra, cont_spectra_arr = apogee_continuum( raw_spectra, raw_spectra_err) self.assertAlmostEqual(np.mean(cont_spectra), 1.)
def test_arXiv_1808_04428(self): """ original astroNN paper models """ from astroNN.apogee import visit_spectra, apogee_continuum from astropy.io import fits # first model models_url = [ "https://github.com/henrysky/astroNN_spectra_paper_figures/trunk/astroNN_0606_run001", "https://github.com/henrysky/astroNN_spectra_paper_figures/trunk/astroNN_0617_run001" ] for model_url in models_url: download_args = ["svn", "export", model_url] res = subprocess.Popen(download_args, stdout=subprocess.PIPE) output, _error = res.communicate() if not _error: pass else: raise ConnectionError( f"Error downloading the models {model_url}") opened_fits = fits.open( visit_spectra(dr=14, location=4405, apogee='2M19060637+4717296')) spectrum = opened_fits[1].data spectrum_err = opened_fits[2].data spectrum_bitmask = opened_fits[3].data # using default continuum and bitmask values to continuum normalize norm_spec, norm_spec_err = apogee_continuum(spectrum, spectrum_err, bitmask=spectrum_bitmask, dr=14) # load neural net neuralnet = load_folder('astroNN_0617_run001') # inference, if there are multiple visits, then you should use the globally # weighted combined spectra (i.e. the second row) pred, pred_err = neuralnet.test(norm_spec) # assert temperature and gravity okay self.assertEqual(np.all(pred[0, 0:2] > [4700., 2.40]), True) self.assertEqual(np.all(pred[0, 0:2] < [4750., 2.47]), True) # load neural net neuralnet = load_folder('astroNN_0606_run001') # inference, if there are multiple visits, then you should use the globally # weighted combined spectra (i.e. the second row) pred, pred_err = neuralnet.test(norm_spec) # assert temperature and gravity okay self.assertEqual(np.all(pred[0, 0:2] > [4700., 2.40]), True) self.assertEqual(np.all(pred[0, 0:2] < [4750., 2.47]), True)
def create_data(self, version=1, max_number_of_stars=float("inf"), use_steps=False): if os.path.exists('{}stars{}.pickle'.format(self._PICKLE_DIR, version)): pickle_out = open( "{}stars{}.pickle".format(self._PICKLE_DIR, version), 'rb') self._star_dict = pickle.load(pickle_out) return True self._star_dict = { 'KIC': [], 'RA': [], 'DEC': [], 'Dnu': [], 'PS': [], 'e_PS': [], 'q': [], 'M': [], 'spectra': [], 'error_spectra': [], 'T_eff': [], 'logg': [], 'RC': [], 'status': [], 'M_H': [], 'C': [], 'N': [], 'O': [], 'Na': [], 'Mg': [], 'Al': [], 'Si': [], 'P': [], 'S': [], 'K': [], 'Ca': [], 'Ti': [], 'V': [], 'Mn': [], 'Fe': [], 'Ni': [], 'Cr': [], 'Co': [], 'Cl': [] } # First create table of Kepler stars with PS and Δv with their RA, DEC (creates table1.dat) self._populate_kepler_with_rc_dec() # Loading in APOGEE star data (RA, DEC) local_path_to_file = allstar(dr=14) apogee_data = fits.open(local_path_to_file) apogee_ra = apogee_data[1].data['RA'] apogee_de = apogee_data[1].data['DEC'] # RA, DEC from KIC table kic_table = Table.read("tables/KIC_A87/table1.dat", format="ascii") kic_ra = np.array(kic_table['RA']) kic_de = np.array(kic_table['DE']) # Indices overlap between KIC and APOGEE idx_kic, idx_apogee, sep = xmatch(kic_ra, apogee_ra, colRA1=kic_ra, colDec1=kic_de, colRA2=apogee_ra, colDec2=apogee_de) # Building spectra data for KIC from APOGEE overlaps for i in range(0, len(idx_apogee)): index_in_apogee = idx_apogee[i] index_in_kepler = idx_kic[i] if version == 1 and not ( apogee_data[1].data['STARFLAG'][index_in_apogee] == 0 and apogee_data[1].data['ASPCAPFLAG'][index_in_apogee] == 0): continue # Version 2 - Subject to the following constraints # Flag checking, condition is the same as Hawkins et al. 2017, STARFLAG and ASPCAP bitwise OR is 0 # KIC checking, uncertainty in PS should be less than 10s if version == 2 and not ( apogee_data[1].data['STARFLAG'][index_in_apogee] == 0 and apogee_data[1].data['ASPCAPFLAG'][index_in_apogee] == 0 and kic_table['e_DPi1'][index_in_kepler] < 10): continue # Version 3 - Subject to the following constraints # Flag checking, condition is the same as Hawkins et al. 2017, STARFLAG and ASPCAP bitwise OR is 0 # KIC checking, uncertainty in PS should be less than 10s if version == 3 and not ( apogee_data[1].data['STARFLAG'][index_in_apogee] == 0 and apogee_data[1].data['ASPCAPFLAG'][index_in_apogee] == 0 and kic_table['e_DPi1'][index_in_kepler] < 10 and apogee_data[1].data['SNR'][index_in_apogee] >= 200): continue # Version 5 - DR13 data subject to same connstraints as Hawkings if version == 5 and not ( apogee_data[1].data['STARFLAG'][index_in_apogee] == 0 and apogee_data[1].data['ASPCAPFLAG'][index_in_apogee] == 0 and kic_table['e_DPi1'][index_in_kepler] < 10): continue # Version 6 - DR13 data subject to same connstraints as Hawkings - normalization with bitmask as DR13 if version == 6 and not ( apogee_data[1].data['STARFLAG'][index_in_apogee] == 0 and apogee_data[1].data['ASPCAPFLAG'][index_in_apogee] == 0 and kic_table['e_DPi1'][index_in_kepler] < 10 and apogee_data[1].data['NVISITS'][index_in_apogee] >= 1): continue # Version 7 - DR13 data subject to same constraints as Hawkings - no bit mask as DR13 if version == 7 and not ( apogee_data[1].data['STARFLAG'][index_in_apogee] == 0 and kic_table['e_DPi1'][index_in_kepler] < 10 and apogee_data[1].data['NVISITS'][index_in_apogee] >= 1): continue # Version 8 - DR13 data subject to same constraints as Hawkings if version == 8 and not ( apogee_data[1].data['STARFLAG'][index_in_apogee] == 0 and kic_table['e_DPi1'][index_in_kepler] < 10 and apogee_data[1].data['NVISITS'][index_in_apogee] >= 1): continue a_apogee_id = apogee_data[1].data['APOGEE_ID'][index_in_apogee] a_location_id = apogee_data[1].data['LOCATION_ID'][index_in_apogee] local_path_to_file_for_star = None if version == 6 or version == 7 or version == 8: local_path_to_file_for_star = visit_spectra( dr=14, location=a_location_id, apogee=a_apogee_id) elif version == 4 or version == 5: local_path_to_file_for_star = combined_spectra( dr=13, location=a_location_id, apogee=a_apogee_id) else: local_path_to_file_for_star = combined_spectra( dr=14, location=a_location_id, apogee=a_apogee_id) if (local_path_to_file_for_star): # Filter out the dr=14 gaps, value to be appended to the array spectra_no_gap = None error_spectra = None if version in (6, 7, 8): # Read from visit spectra of the star spectra_data = fits.open(local_path_to_file_for_star) spectra, mask_spectra = None, None # Best fit spectra data - use for spectra data if apogee_data[1].data['NVISITS'][index_in_apogee] == 1: spectra = spectra_data[1].data error_spectra = spectra_data[2].data mask_spectra = spectra_data[3].data elif apogee_data[1].data['NVISITS'][index_in_apogee] > 1: spectra = spectra_data[1].data[1] error_spectra = spectra_data[2].data[1] mask_spectra = spectra_data[3].data[1] if version == 6: norm_spec, norm_spec_err = apogee_continuum( spectra, error_spectra, cont_mask=None, deg=2, dr=13, bitmask=mask_spectra, target_bit=None) spectra_no_gap = norm_spec error_spectra = norm_spec_err elif version == 7: norm_spec, norm_spec_err = apogee_continuum( spectra, error_spectra, cont_mask=None, deg=2, dr=13, bitmask=None, target_bit=None) spectra_no_gap = norm_spec error_spectra = norm_spec_err elif version == 8: # Bit mask value is now 1 norm_spec, norm_spec_err = apogee_continuum( spectra, error_spectra, cont_mask=None, deg=2, dr=13, bitmask=mask_spectra, target_bit=None, mask_value=1) spectra_no_gap = norm_spec error_spectra = norm_spec_err del mask_spectra elif version == 4 or version == 5: spectra_data = fits.open(local_path_to_file_for_star) # Best fit spectra data - use for spectra data spectra = spectra_data[3].data spectra_no_gap = gap_delete(spectra, dr=13) else: spectra_data = fits.open(local_path_to_file_for_star) # Best fit spectra data - use for spectra data spectra = spectra_data[3].data spectra_no_gap = gap_delete(spectra, dr=14) spectra_no_gap = spectra_no_gap.flatten() # APOGEE data self._star_dict['T_eff'].append( apogee_data[1].data['Teff'][index_in_apogee].copy()) self._star_dict['logg'].append( apogee_data[1].data['logg'][index_in_apogee].copy()) # Metals/H self._star_dict['M_H'].append( apogee_data[1].data['M_H'][index_in_apogee]) self._star_dict['C'].append( apogee_data[1].data['X_H'][index_in_apogee][0]) self._star_dict['N'].append( apogee_data[1].data['X_H'][index_in_apogee][2]) self._star_dict['O'].append( apogee_data[1].data['X_H'][index_in_apogee][3]) self._star_dict['Na'].append( apogee_data[1].data['X_H'][index_in_apogee][4]) self._star_dict['Mg'].append( apogee_data[1].data['X_H'][index_in_apogee][5]) self._star_dict['Al'].append( apogee_data[1].data['X_H'][index_in_apogee][6]) self._star_dict['Si'].append( apogee_data[1].data['X_H'][index_in_apogee][7]) self._star_dict['P'].append( apogee_data[1].data['X_H'][index_in_apogee][8]) self._star_dict['S'].append( apogee_data[1].data['X_H'][index_in_apogee][9]) self._star_dict['K'].append( apogee_data[1].data['X_H'][index_in_apogee][10]) self._star_dict['Ca'].append( apogee_data[1].data['X_H'][index_in_apogee][11]) self._star_dict['Ti'].append( apogee_data[1].data['X_H'][index_in_apogee][12]) self._star_dict['V'].append( apogee_data[1].data['X_H'][index_in_apogee][14]) self._star_dict['Mn'].append( apogee_data[1].data['X_H'][index_in_apogee][16]) self._star_dict['Fe'].append( apogee_data[1].data['X_H'][index_in_apogee][17]) self._star_dict['Ni'].append( apogee_data[1].data['X_H'][index_in_apogee][19]) self._star_dict['Cr'].append( apogee_data[1].data['X_H'][index_in_apogee][15]) self._star_dict['Co'].append( apogee_data[1].data['X_H'][index_in_apogee][18]) self._star_dict['Cl'].append( apogee_data[1].data['X_H'][index_in_apogee][1]) # KIC data self._star_dict['KIC'].append( kic_table['KIC'][index_in_kepler]) self._star_dict['RA'].append(kic_table['RA'][index_in_kepler]) self._star_dict['DEC'].append(kic_table['DE'][index_in_kepler]) self._star_dict['Dnu'].append( kic_table['Dnu'][index_in_kepler]) self._star_dict['PS'].append( kic_table['DPi1'][index_in_kepler]) self._star_dict['e_PS'].append( kic_table['e_DPi1'][index_in_kepler]) self._star_dict['q'].append(kic_table['q'][index_in_kepler]) self._star_dict['M'].append(kic_table['M'][index_in_kepler]) self._star_dict['status'].append( kic_table['Status'][index_in_kepler]) self._star_dict['RC'].append( self.is_red_clump(kic_table['DPi1'][index_in_kepler], kic_table['Dnu'][index_in_kepler])) # Gap delete doesn't return row vector, need to manually reshape self._star_dict['spectra'].append(spectra_no_gap) if version in (6, 7, 8): self._star_dict['error_spectra'].append( error_spectra.flatten()) del error_spectra # Close file handler del spectra_data del spectra # Check max condition if i > max_number_of_stars - 1: break # Convert to numpy arrays self._star_dict['KIC'] = np.array(self._star_dict['KIC']) self._star_dict['RA'] = np.array(self._star_dict['RA']) self._star_dict['DEC'] = np.array(self._star_dict['DEC']) self._star_dict['Dnu'] = np.array(self._star_dict['Dnu']) self._star_dict['PS'] = np.array(self._star_dict['PS']) self._star_dict['e_PS'] = np.array(self._star_dict['e_PS']) self._star_dict['spectra'] = np.array(self._star_dict['spectra']) self._star_dict['logg'] = np.array(self._star_dict['logg']) self._star_dict['T_eff'] = np.array(self._star_dict['T_eff']) self._star_dict['RC'] = np.array(self._star_dict['RC']) self._star_dict['status'] = np.array(self._star_dict['status']) self._star_dict['M'] = np.array(self._star_dict['M']) self._star_dict['M_H'] = np.array(self._star_dict['M_H']) self._star_dict['C'] = np.array(self._star_dict['C']) self._star_dict['N'] = np.array(self._star_dict['N']) self._star_dict['O'] = np.array(self._star_dict['O']) self._star_dict['Na'] = np.array(self._star_dict['Na']) self._star_dict['Mg'] = np.array(self._star_dict['Mg']) self._star_dict['Al'] = np.array(self._star_dict['Al']) self._star_dict['Si'] = np.array(self._star_dict['Si']) self._star_dict['P'] = np.array(self._star_dict['P']) self._star_dict['S'] = np.array(self._star_dict['S']) self._star_dict['K'] = np.array(self._star_dict['K']) self._star_dict['Ca'] = np.array(self._star_dict['Ca']) self._star_dict['Ti'] = np.array(self._star_dict['Ti']) self._star_dict['V'] = np.array(self._star_dict['V']) self._star_dict['Mn'] = np.array(self._star_dict['Mn']) self._star_dict['Fe'] = np.array(self._star_dict['Fe']) self._star_dict['Ni'] = np.array(self._star_dict['Ni']) self._star_dict['Cl'] = np.array(self._star_dict['Cl']) self._star_dict['Cr'] = np.array(self._star_dict['Cr']) self._star_dict['Co'] = np.array(self._star_dict['Co']) self._star_dict['q'] = np.array(self._star_dict['q']) if version == 6: self._star_dict['error_spectra'] = np.array( self._star_dict['error_spectra']) # Pickle for caching pickle_out = open("{}stars{}.pickle".format(self._PICKLE_DIR, version), 'wb') pickle.dump(self._star_dict, pickle_out) pickle_out.close() return True
def grab_sample(self, N = 32, dr=14, spectra_format='visit', bitmask_value =1, save_sample=True): self.create_data() stars = { 'APSTAR_ID' : [], # Spectra of the star 'spectra' : [], # Stellar features 'Teff' : [], 'logg' : [], # Metals 'M_H' : [], 'C' : [], 'N' : [], 'O' : [], 'Na' : [], 'Mg' : [], 'Al' : [], 'Si' : [], 'P' : [], 'S' : [], 'K' : [], 'Ca' : [], 'Ti' : [], 'V' : [], 'Mn' : [], 'Fe' : [], 'Ni' : [], } # Get random indices idx = np.random.choice(len(self.apogee_data[1].data['logg']), len(self.apogee_data[1].data['logg'])) for i in idx: # Break condition if len(stars['logg']) >= N: break # For when the spectra_format is visit, we require more than NVISIT >= 1 if spectra_format == 'visit' and (self.apogee_data[1].data['NVISITS'][i] < 1): continue # Enforce we have high quality spectra from dr14 if not(self.apogee_data[1].data['STARFLAG'][i] == 0 and self.apogee_data[1].data['ASPCAPFLAG'][i] == 0 and self.apogee_data[1].data['SNR'][i] < 200): # Stellar features logg = self.apogee_data[1].data['logg'][i] Teff = self.apogee_data[1].data['Teff'][i] # Metals/H M_H = self.apogee_data[1].data['M_H'][i] # Metals C = self.apogee_data[1].data['X_H'][i][0] N_c = self.apogee_data[1].data['X_H'][i][2] O = self.apogee_data[1].data['X_H'][i][3] Na = self.apogee_data[1].data['X_H'][i][4] Mg = self.apogee_data[1].data['X_H'][i][5] Al = self.apogee_data[1].data['X_H'][i][6] Si = self.apogee_data[1].data['X_H'][i][7] P = self.apogee_data[1].data['X_H'][i][8] S = self.apogee_data[1].data['X_H'][i][9] K = self.apogee_data[1].data['X_H'][i][10] Ca = self.apogee_data[1].data['X_H'][i][11] Ti = self.apogee_data[1].data['X_H'][i][12] V = self.apogee_data[1].data['X_H'][i][14] Mn = self.apogee_data[1].data['X_H'][i][16] Fe = self.apogee_data[1].data['X_H'][i][17] Ni = self.apogee_data[1].data['X_H'][i][19] # Make sure all of them are not falled if all([False if i == -9999 else True for i in [logg, Teff, M_H, C, N_c, O, Na, Mg, Al, Si, P, S, K, Ca, Ti, V, Mn, Fe, Ni]]): # Get spectra data apogee_id, location_id = self.apogee_data[1].data['APOGEE_ID'][i], self.apogee_data[1].data['LOCATION_ID'][i] star_spectra = None # Populating the appropiate star_spectra variable with stellar spectra if spectra_format == 'combined': local_path_to_file_for_star = combined_spectra(dr=dr, location=location_id, apogee=apogee_id) if local_path_to_file_for_star: spectra_data = fits.open(local_path_to_file_for_star) star_spectra = spectra_data[3].data.copy() star_spectra = gap_delete(spectra, dr=dr) star_spectra = star_spectra.flatten() # Close file handlers del spectra_data elif spectra_format == 'visit': local_path_to_file_for_star = visit_spectra(dr=dr, location=location_id, apogee=apogee_id) if local_path_to_file_for_star: spectra_data = fits.open(local_path_to_file_for_star) spectra, error_spectra, mask_spectra = None, None, None # NVISITS is 1, 1D array if self.apogee_data[1].data['NVISITS'][i] == 1: spectra = spectra_data[1].data error_spectra = spectra_data[2].data mask_spectra = spectra_data[3].data elif self.apogee_data[1].data['NVISITS'][i] > 1: spectra = spectra_data[1].data[1] error_spectra = spectra_data[2].data[1] mask_spectra = spectra_data[3].data[1] # Mask if the mask value is present if bitmask_value is not None: norm_spec, norm_spec_err = apogee_continuum(spectra, error_spectra, cont_mask=None, deg=2, bitmask=mask_spectra, target_bit=None, mask_value=bitmask_value) star_spectra = norm_spec.flatten() else: norm_spec, norm_spec_err = apogee_continuum(spectra, error_spectra, cont_mask=None, deg=2, bitmask=None, target_bit=None, mask_value=None) star_spectra = norm_spec.flatten() # Close file handlers del spectra_data else: raise ValueError("spectra_format must either be combined|visit") # Adding the star data into the dict if star_spectra is not None: stars['spectra'].append(star_spectra) stars['logg'].append(logg) stars['Teff'].append(Teff) stars['M_H'].append(M_H) stars['C'].append(C) stars['N'].append(N_c) stars['O'].append(O) stars['Na'].append(Na) stars['Mg'].append(Mg) stars['Al'].append(Al) stars['Si'].append(Si) stars['P'].append(P) stars['S'].append(S) stars['K'].append(K) stars['Ca'].append(Ca) stars['Ti'].append(Ti) stars['V'].append(V) stars['Mn'].append(Mn) stars['Fe'].append(Fe) stars['Ni'].append(Ni) # Convert to np style arrays for key in stars: stars[key] = np.array(stars[key]) # stars['logg'] = np.array(stars['logg']) # stars['Teff'] = np.array(stars['Teff']) # stars['M_H'] = np.array(stars['M_H']) # stars['C'] = np.array(stars['C']) # stars['N'] = np.array(stars['N']) # stars['O'] = np.array(stars['O']) # stars['Na'] = np.array(stars['Na']) # stars['Mg'] = np.array(stars['Mg']) # stars['Al'] = np.array(stars['Al']) # stars['Si'] = np.array(stars['Si']) # stars['P'] = np.array(stars['P']) # stars['S'] = np.array(stars['S']) # stars['K'] = np.array(stars['K']) # stars['Ca'] = np.array(stars['Ca']) # stars['Ti'] = np.array(stars['Ti']) # stars['V'] = np.array(stars['V']) # stars['Mn'] = np.array(stars['Mn']) # stars['Fe'] = np.array(stars['Fe']) # stars['Ni'] = np.array(stars['Ni']) # stars['spectra'] = np.array(stars['spectra']) # Save sample if asked to if save_sample: pickle_out = open("{}apogee_sample_{}_stars.pickle".format(self._PICKLE_DIR, N), 'wb') pickle.dump(stars, pickle_out) pickle_out.close() return stars
def test_arXiv_1902_08634 (self): """ astroNN spectrophotometric distance """ from astroNN.apogee import visit_spectra, apogee_continuum from astroNN.gaia import extinction_correction, fakemag_to_pc from astropy.io import fits # first model models_url = ["https://github.com/henrysky/astroNN_gaia_dr2_paper/trunk/astroNN_no_offset_model", "https://github.com/henrysky/astroNN_gaia_dr2_paper/trunk/astroNN_constant_model", "https://github.com/henrysky/astroNN_gaia_dr2_paper/trunk/astroNN_multivariate_model"] for model_url in models_url: download_args = ["svn", "export", model_url] res = subprocess.Popen(download_args, stdout=subprocess.PIPE) output, _error = res.communicate() if not _error: pass else: raise ConnectionError(f"Error downloading the models {model_url}") opened_fits = fits.open(visit_spectra(dr=14, location=4405, apogee='2M19060637+4717296')) spectrum = opened_fits[1].data spectrum_err = opened_fits[2].data spectrum_bitmask = opened_fits[3].data # using default continuum and bitmask values to continuum normalize norm_spec, norm_spec_err = apogee_continuum(spectrum, spectrum_err, bitmask=spectrum_bitmask, dr=14) # correct for extinction K = extinction_correction(opened_fits[0].header['K'], opened_fits[0].header['AKTARG']) # ===========================================================================================# # load neural net neuralnet = load_folder('astroNN_no_offset_model') # inference, if there are multiple visits, then you should use the globally # weighted combined spectra (i.e. the second row) pred, pred_err = neuralnet.test(norm_spec) # convert prediction in fakemag to distance pc, pc_error = fakemag_to_pc(pred[:, 0], K, pred_err['total'][:, 0]) # assert distance is close enough # http://simbad.u-strasbg.fr/simbad/sim-id?mescat.distance=on&Ident=%406876647&Name=KIC+10196240&submit=display+selected+measurements#lab_meas # no offset correction so further away self.assertEqual(pc.value < 1250, True) self.assertEqual(pc.value > 1100, True) # ===========================================================================================# # load neural net neuralnet = load_folder('astroNN_constant_model') # inference, if there are multiple visits, then you should use the globally # weighted combined spectra (i.e. the second row) pred, pred_err = neuralnet.test(np.hstack([norm_spec, np.zeros((norm_spec.shape[0], 4))])) # convert prediction in fakemag to distance pc, pc_error = fakemag_to_pc(pred[:, 0], K, pred_err['total'][:, 0]) # assert distance is close enough # http://simbad.u-strasbg.fr/simbad/sim-id?mescat.distance=on&Ident=%406876647&Name=KIC+10196240&submit=display+selected+measurements#lab_meas self.assertEqual(pc.value < 1150, True) self.assertEqual(pc.value > 1000, True) # ===========================================================================================# # load neural net neuralnet = load_folder('astroNN_multivariate_model') # inference, if there are multiple visits, then you should use the globally # weighted combined spectra (i.e. the second row) pred, pred_err = neuralnet.test(np.hstack([norm_spec, np.zeros((norm_spec.shape[0], 4))])) # convert prediction in fakemag to distance pc, pc_error = fakemag_to_pc(pred[:, 0], K, pred_err['total'][:, 0]) # assert distance is close enough # http://simbad.u-strasbg.fr/simbad/sim-id?mescat.distance=on&Ident=%406876647&Name=KIC+10196240&submit=display+selected+measurements#lab_meas self.assertEqual(pc.value < 1150, True) self.assertEqual(pc.value > 1000, True)