def test_apogee_combined_download(self):
combined_spectra(dr=13, location=4405, apogee='2M19060637+4717296')
combined_spectra(dr=14, location=4405, apogee='2M19060637+4717296')
self.assertEqual(
combined_spectra(dr=13, location=4406,
apogee='2M19060637+4717296'), False)
self.assertEqual(
combined_spectra(dr=14, location=4406,
apogee='2M19060637+4717296'), False)
self.assertRaises(ValueError,
combined_spectra,
dr=1,
location=4406,
apogee='2M19060637+4717296')
def test_apogee_combined_download(self):
"""
Test APOGEE combined spectra downloading function, assert functions can deal with missing files
"""
# make sure the download works correctly
combined_spectra(dr=13, location=4405, apogee='2M19060637+4717296')
combined_spectra(dr=14, location=4405, apogee='2M19060637+4717296')
# assert False is returning if file not found
self.assertEqual(
combined_spectra(dr=13, location=4406,
apogee='2M19060637+4717296'), False)
self.assertEqual(
combined_spectra(dr=14, location=4406,
apogee='2M19060637+4717296'), False)
# assert error if DR not supported
self.assertRaises(ValueError,
combined_spectra,
dr=1,
location=4406,
apogee='2M19060637+4717296')
def create_data(self,
version=1,
max_number_of_stars=float("inf"),
use_steps=False):
if os.path.exists('{}stars{}.pickle'.format(self._PICKLE_DIR,
version)):
pickle_out = open(
"{}stars{}.pickle".format(self._PICKLE_DIR, version), 'rb')
self._star_dict = pickle.load(pickle_out)
return True
self._star_dict = {
'KIC': [],
'RA': [],
'DEC': [],
'Dnu': [],
'PS': [],
'e_PS': [],
'q': [],
'M': [],
'spectra': [],
'error_spectra': [],
'T_eff': [],
'logg': [],
'RC': [],
'status': [],
'M_H': [],
'C': [],
'N': [],
'O': [],
'Na': [],
'Mg': [],
'Al': [],
'Si': [],
'P': [],
'S': [],
'K': [],
'Ca': [],
'Ti': [],
'V': [],
'Mn': [],
'Fe': [],
'Ni': [],
'Cr': [],
'Co': [],
'Cl': []
}
# First create table of Kepler stars with PS and Δv with their RA, DEC (creates table1.dat)
self._populate_kepler_with_rc_dec()
# Loading in APOGEE star data (RA, DEC)
local_path_to_file = allstar(dr=14)
apogee_data = fits.open(local_path_to_file)
apogee_ra = apogee_data[1].data['RA']
apogee_de = apogee_data[1].data['DEC']
# RA, DEC from KIC table
kic_table = Table.read("tables/KIC_A87/table1.dat", format="ascii")
kic_ra = np.array(kic_table['RA'])
kic_de = np.array(kic_table['DE'])
# Indices overlap between KIC and APOGEE
idx_kic, idx_apogee, sep = xmatch(kic_ra,
apogee_ra,
colRA1=kic_ra,
colDec1=kic_de,
colRA2=apogee_ra,
colDec2=apogee_de)
# Building spectra data for KIC from APOGEE overlaps
for i in range(0, len(idx_apogee)):
index_in_apogee = idx_apogee[i]
index_in_kepler = idx_kic[i]
if version == 1 and not (
apogee_data[1].data['STARFLAG'][index_in_apogee] == 0 and
apogee_data[1].data['ASPCAPFLAG'][index_in_apogee] == 0):
continue
# Version 2 - Subject to the following constraints
# Flag checking, condition is the same as Hawkins et al. 2017, STARFLAG and ASPCAP bitwise OR is 0
# KIC checking, uncertainty in PS should be less than 10s
if version == 2 and not (
apogee_data[1].data['STARFLAG'][index_in_apogee] == 0
and apogee_data[1].data['ASPCAPFLAG'][index_in_apogee] == 0
and kic_table['e_DPi1'][index_in_kepler] < 10):
continue
# Version 3 - Subject to the following constraints
# Flag checking, condition is the same as Hawkins et al. 2017, STARFLAG and ASPCAP bitwise OR is 0
# KIC checking, uncertainty in PS should be less than 10s
if version == 3 and not (
apogee_data[1].data['STARFLAG'][index_in_apogee] == 0
and apogee_data[1].data['ASPCAPFLAG'][index_in_apogee] == 0
and kic_table['e_DPi1'][index_in_kepler] < 10
and apogee_data[1].data['SNR'][index_in_apogee] >= 200):
continue
# Version 5 - DR13 data subject to same connstraints as Hawkings
if version == 5 and not (
apogee_data[1].data['STARFLAG'][index_in_apogee] == 0
and apogee_data[1].data['ASPCAPFLAG'][index_in_apogee] == 0
and kic_table['e_DPi1'][index_in_kepler] < 10):
continue
# Version 6 - DR13 data subject to same connstraints as Hawkings - normalization with bitmask as DR13
if version == 6 and not (
apogee_data[1].data['STARFLAG'][index_in_apogee] == 0
and apogee_data[1].data['ASPCAPFLAG'][index_in_apogee] == 0
and kic_table['e_DPi1'][index_in_kepler] < 10
and apogee_data[1].data['NVISITS'][index_in_apogee] >= 1):
continue
# Version 7 - DR13 data subject to same constraints as Hawkings - no bit mask as DR13
if version == 7 and not (
apogee_data[1].data['STARFLAG'][index_in_apogee] == 0
and kic_table['e_DPi1'][index_in_kepler] < 10
and apogee_data[1].data['NVISITS'][index_in_apogee] >= 1):
continue
# Version 8 - DR13 data subject to same constraints as Hawkings
if version == 8 and not (
apogee_data[1].data['STARFLAG'][index_in_apogee] == 0
and kic_table['e_DPi1'][index_in_kepler] < 10
and apogee_data[1].data['NVISITS'][index_in_apogee] >= 1):
continue
a_apogee_id = apogee_data[1].data['APOGEE_ID'][index_in_apogee]
a_location_id = apogee_data[1].data['LOCATION_ID'][index_in_apogee]
local_path_to_file_for_star = None
if version == 6 or version == 7 or version == 8:
local_path_to_file_for_star = visit_spectra(
dr=14, location=a_location_id, apogee=a_apogee_id)
elif version == 4 or version == 5:
local_path_to_file_for_star = combined_spectra(
dr=13, location=a_location_id, apogee=a_apogee_id)
else:
local_path_to_file_for_star = combined_spectra(
dr=14, location=a_location_id, apogee=a_apogee_id)
if (local_path_to_file_for_star):
# Filter out the dr=14 gaps, value to be appended to the array
spectra_no_gap = None
error_spectra = None
if version in (6, 7, 8):
# Read from visit spectra of the star
spectra_data = fits.open(local_path_to_file_for_star)
spectra, mask_spectra = None, None
# Best fit spectra data - use for spectra data
if apogee_data[1].data['NVISITS'][index_in_apogee] == 1:
spectra = spectra_data[1].data
error_spectra = spectra_data[2].data
mask_spectra = spectra_data[3].data
elif apogee_data[1].data['NVISITS'][index_in_apogee] > 1:
spectra = spectra_data[1].data[1]
error_spectra = spectra_data[2].data[1]
mask_spectra = spectra_data[3].data[1]
if version == 6:
norm_spec, norm_spec_err = apogee_continuum(
spectra,
error_spectra,
cont_mask=None,
deg=2,
dr=13,
bitmask=mask_spectra,
target_bit=None)
spectra_no_gap = norm_spec
error_spectra = norm_spec_err
elif version == 7:
norm_spec, norm_spec_err = apogee_continuum(
spectra,
error_spectra,
cont_mask=None,
deg=2,
dr=13,
bitmask=None,
target_bit=None)
spectra_no_gap = norm_spec
error_spectra = norm_spec_err
elif version == 8:
# Bit mask value is now 1
norm_spec, norm_spec_err = apogee_continuum(
spectra,
error_spectra,
cont_mask=None,
deg=2,
dr=13,
bitmask=mask_spectra,
target_bit=None,
mask_value=1)
spectra_no_gap = norm_spec
error_spectra = norm_spec_err
del mask_spectra
elif version == 4 or version == 5:
spectra_data = fits.open(local_path_to_file_for_star)
# Best fit spectra data - use for spectra data
spectra = spectra_data[3].data
spectra_no_gap = gap_delete(spectra, dr=13)
else:
spectra_data = fits.open(local_path_to_file_for_star)
# Best fit spectra data - use for spectra data
spectra = spectra_data[3].data
spectra_no_gap = gap_delete(spectra, dr=14)
spectra_no_gap = spectra_no_gap.flatten()
# APOGEE data
self._star_dict['T_eff'].append(
apogee_data[1].data['Teff'][index_in_apogee].copy())
self._star_dict['logg'].append(
apogee_data[1].data['logg'][index_in_apogee].copy())
# Metals/H
self._star_dict['M_H'].append(
apogee_data[1].data['M_H'][index_in_apogee])
self._star_dict['C'].append(
apogee_data[1].data['X_H'][index_in_apogee][0])
self._star_dict['N'].append(
apogee_data[1].data['X_H'][index_in_apogee][2])
self._star_dict['O'].append(
apogee_data[1].data['X_H'][index_in_apogee][3])
self._star_dict['Na'].append(
apogee_data[1].data['X_H'][index_in_apogee][4])
self._star_dict['Mg'].append(
apogee_data[1].data['X_H'][index_in_apogee][5])
self._star_dict['Al'].append(
apogee_data[1].data['X_H'][index_in_apogee][6])
self._star_dict['Si'].append(
apogee_data[1].data['X_H'][index_in_apogee][7])
self._star_dict['P'].append(
apogee_data[1].data['X_H'][index_in_apogee][8])
self._star_dict['S'].append(
apogee_data[1].data['X_H'][index_in_apogee][9])
self._star_dict['K'].append(
apogee_data[1].data['X_H'][index_in_apogee][10])
self._star_dict['Ca'].append(
apogee_data[1].data['X_H'][index_in_apogee][11])
self._star_dict['Ti'].append(
apogee_data[1].data['X_H'][index_in_apogee][12])
self._star_dict['V'].append(
apogee_data[1].data['X_H'][index_in_apogee][14])
self._star_dict['Mn'].append(
apogee_data[1].data['X_H'][index_in_apogee][16])
self._star_dict['Fe'].append(
apogee_data[1].data['X_H'][index_in_apogee][17])
self._star_dict['Ni'].append(
apogee_data[1].data['X_H'][index_in_apogee][19])
self._star_dict['Cr'].append(
apogee_data[1].data['X_H'][index_in_apogee][15])
self._star_dict['Co'].append(
apogee_data[1].data['X_H'][index_in_apogee][18])
self._star_dict['Cl'].append(
apogee_data[1].data['X_H'][index_in_apogee][1])
# KIC data
self._star_dict['KIC'].append(
kic_table['KIC'][index_in_kepler])
self._star_dict['RA'].append(kic_table['RA'][index_in_kepler])
self._star_dict['DEC'].append(kic_table['DE'][index_in_kepler])
self._star_dict['Dnu'].append(
kic_table['Dnu'][index_in_kepler])
self._star_dict['PS'].append(
kic_table['DPi1'][index_in_kepler])
self._star_dict['e_PS'].append(
kic_table['e_DPi1'][index_in_kepler])
self._star_dict['q'].append(kic_table['q'][index_in_kepler])
self._star_dict['M'].append(kic_table['M'][index_in_kepler])
self._star_dict['status'].append(
kic_table['Status'][index_in_kepler])
self._star_dict['RC'].append(
self.is_red_clump(kic_table['DPi1'][index_in_kepler],
kic_table['Dnu'][index_in_kepler]))
# Gap delete doesn't return row vector, need to manually reshape
self._star_dict['spectra'].append(spectra_no_gap)
if version in (6, 7, 8):
self._star_dict['error_spectra'].append(
error_spectra.flatten())
del error_spectra
# Close file handler
del spectra_data
del spectra
# Check max condition
if i > max_number_of_stars - 1:
break
# Convert to numpy arrays
self._star_dict['KIC'] = np.array(self._star_dict['KIC'])
self._star_dict['RA'] = np.array(self._star_dict['RA'])
self._star_dict['DEC'] = np.array(self._star_dict['DEC'])
self._star_dict['Dnu'] = np.array(self._star_dict['Dnu'])
self._star_dict['PS'] = np.array(self._star_dict['PS'])
self._star_dict['e_PS'] = np.array(self._star_dict['e_PS'])
self._star_dict['spectra'] = np.array(self._star_dict['spectra'])
self._star_dict['logg'] = np.array(self._star_dict['logg'])
self._star_dict['T_eff'] = np.array(self._star_dict['T_eff'])
self._star_dict['RC'] = np.array(self._star_dict['RC'])
self._star_dict['status'] = np.array(self._star_dict['status'])
self._star_dict['M'] = np.array(self._star_dict['M'])
self._star_dict['M_H'] = np.array(self._star_dict['M_H'])
self._star_dict['C'] = np.array(self._star_dict['C'])
self._star_dict['N'] = np.array(self._star_dict['N'])
self._star_dict['O'] = np.array(self._star_dict['O'])
self._star_dict['Na'] = np.array(self._star_dict['Na'])
self._star_dict['Mg'] = np.array(self._star_dict['Mg'])
self._star_dict['Al'] = np.array(self._star_dict['Al'])
self._star_dict['Si'] = np.array(self._star_dict['Si'])
self._star_dict['P'] = np.array(self._star_dict['P'])
self._star_dict['S'] = np.array(self._star_dict['S'])
self._star_dict['K'] = np.array(self._star_dict['K'])
self._star_dict['Ca'] = np.array(self._star_dict['Ca'])
self._star_dict['Ti'] = np.array(self._star_dict['Ti'])
self._star_dict['V'] = np.array(self._star_dict['V'])
self._star_dict['Mn'] = np.array(self._star_dict['Mn'])
self._star_dict['Fe'] = np.array(self._star_dict['Fe'])
self._star_dict['Ni'] = np.array(self._star_dict['Ni'])
self._star_dict['Cl'] = np.array(self._star_dict['Cl'])
self._star_dict['Cr'] = np.array(self._star_dict['Cr'])
self._star_dict['Co'] = np.array(self._star_dict['Co'])
self._star_dict['q'] = np.array(self._star_dict['q'])
if version == 6:
self._star_dict['error_spectra'] = np.array(
self._star_dict['error_spectra'])
# Pickle for caching
pickle_out = open("{}stars{}.pickle".format(self._PICKLE_DIR, version),
'wb')
pickle.dump(self._star_dict, pickle_out)
pickle_out.close()
return True
def grab_sample(self, N = 32, dr=14, spectra_format='visit', bitmask_value =1, save_sample=True):
self.create_data()
stars = {
'APSTAR_ID' : [],
# Spectra of the star
'spectra' : [],
# Stellar features
'Teff' : [],
'logg' : [],
# Metals
'M_H' : [],
'C' : [],
'N' : [],
'O' : [],
'Na' : [],
'Mg' : [],
'Al' : [],
'Si' : [],
'P' : [],
'S' : [],
'K' : [],
'Ca' : [],
'Ti' : [],
'V' : [],
'Mn' : [],
'Fe' : [],
'Ni' : [],
}
# Get random indices
idx = np.random.choice(len(self.apogee_data[1].data['logg']), len(self.apogee_data[1].data['logg']))
for i in idx:
# Break condition
if len(stars['logg']) >= N:
break
# For when the spectra_format is visit, we require more than NVISIT >= 1
if spectra_format == 'visit' and (self.apogee_data[1].data['NVISITS'][i] < 1):
continue
# Enforce we have high quality spectra from dr14
if not(self.apogee_data[1].data['STARFLAG'][i] == 0 and self.apogee_data[1].data['ASPCAPFLAG'][i] == 0 and self.apogee_data[1].data['SNR'][i] < 200):
# Stellar features
logg = self.apogee_data[1].data['logg'][i]
Teff = self.apogee_data[1].data['Teff'][i]
# Metals/H
M_H = self.apogee_data[1].data['M_H'][i]
# Metals
C = self.apogee_data[1].data['X_H'][i][0]
N_c = self.apogee_data[1].data['X_H'][i][2]
O = self.apogee_data[1].data['X_H'][i][3]
Na = self.apogee_data[1].data['X_H'][i][4]
Mg = self.apogee_data[1].data['X_H'][i][5]
Al = self.apogee_data[1].data['X_H'][i][6]
Si = self.apogee_data[1].data['X_H'][i][7]
P = self.apogee_data[1].data['X_H'][i][8]
S = self.apogee_data[1].data['X_H'][i][9]
K = self.apogee_data[1].data['X_H'][i][10]
Ca = self.apogee_data[1].data['X_H'][i][11]
Ti = self.apogee_data[1].data['X_H'][i][12]
V = self.apogee_data[1].data['X_H'][i][14]
Mn = self.apogee_data[1].data['X_H'][i][16]
Fe = self.apogee_data[1].data['X_H'][i][17]
Ni = self.apogee_data[1].data['X_H'][i][19]
# Make sure all of them are not falled
if all([False if i == -9999 else True for i in [logg, Teff, M_H, C, N_c, O, Na, Mg, Al, Si, P, S, K, Ca, Ti, V, Mn, Fe, Ni]]):
# Get spectra data
apogee_id, location_id = self.apogee_data[1].data['APOGEE_ID'][i], self.apogee_data[1].data['LOCATION_ID'][i]
star_spectra = None
# Populating the appropiate star_spectra variable with stellar spectra
if spectra_format == 'combined':
local_path_to_file_for_star = combined_spectra(dr=dr, location=location_id, apogee=apogee_id)
if local_path_to_file_for_star:
spectra_data = fits.open(local_path_to_file_for_star)
star_spectra = spectra_data[3].data.copy()
star_spectra = gap_delete(spectra, dr=dr)
star_spectra = star_spectra.flatten()
# Close file handlers
del spectra_data
elif spectra_format == 'visit':
local_path_to_file_for_star = visit_spectra(dr=dr, location=location_id, apogee=apogee_id)
if local_path_to_file_for_star:
spectra_data = fits.open(local_path_to_file_for_star)
spectra, error_spectra, mask_spectra = None, None, None
# NVISITS is 1, 1D array
if self.apogee_data[1].data['NVISITS'][i] == 1:
spectra = spectra_data[1].data
error_spectra = spectra_data[2].data
mask_spectra = spectra_data[3].data
elif self.apogee_data[1].data['NVISITS'][i] > 1:
spectra = spectra_data[1].data[1]
error_spectra = spectra_data[2].data[1]
mask_spectra = spectra_data[3].data[1]
# Mask if the mask value is present
if bitmask_value is not None:
norm_spec, norm_spec_err = apogee_continuum(spectra, error_spectra, cont_mask=None, deg=2, bitmask=mask_spectra, target_bit=None, mask_value=bitmask_value)
star_spectra = norm_spec.flatten()
else:
norm_spec, norm_spec_err = apogee_continuum(spectra, error_spectra, cont_mask=None, deg=2, bitmask=None, target_bit=None, mask_value=None)
star_spectra = norm_spec.flatten()
# Close file handlers
del spectra_data
else:
raise ValueError("spectra_format must either be combined|visit")
# Adding the star data into the dict
if star_spectra is not None:
stars['spectra'].append(star_spectra)
stars['logg'].append(logg)
stars['Teff'].append(Teff)
stars['M_H'].append(M_H)
stars['C'].append(C)
stars['N'].append(N_c)
stars['O'].append(O)
stars['Na'].append(Na)
stars['Mg'].append(Mg)
stars['Al'].append(Al)
stars['Si'].append(Si)
stars['P'].append(P)
stars['S'].append(S)
stars['K'].append(K)
stars['Ca'].append(Ca)
stars['Ti'].append(Ti)
stars['V'].append(V)
stars['Mn'].append(Mn)
stars['Fe'].append(Fe)
stars['Ni'].append(Ni)
# Convert to np style arrays
for key in stars:
stars[key] = np.array(stars[key])
# stars['logg'] = np.array(stars['logg'])
# stars['Teff'] = np.array(stars['Teff'])
# stars['M_H'] = np.array(stars['M_H'])
# stars['C'] = np.array(stars['C'])
# stars['N'] = np.array(stars['N'])
# stars['O'] = np.array(stars['O'])
# stars['Na'] = np.array(stars['Na'])
# stars['Mg'] = np.array(stars['Mg'])
# stars['Al'] = np.array(stars['Al'])
# stars['Si'] = np.array(stars['Si'])
# stars['P'] = np.array(stars['P'])
# stars['S'] = np.array(stars['S'])
# stars['K'] = np.array(stars['K'])
# stars['Ca'] = np.array(stars['Ca'])
# stars['Ti'] = np.array(stars['Ti'])
# stars['V'] = np.array(stars['V'])
# stars['Mn'] = np.array(stars['Mn'])
# stars['Fe'] = np.array(stars['Fe'])
# stars['Ni'] = np.array(stars['Ni'])
# stars['spectra'] = np.array(stars['spectra'])
# Save sample if asked to
if save_sample:
pickle_out = open("{}apogee_sample_{}_stars.pickle".format(self._PICKLE_DIR, N), 'wb')
pickle.dump(stars, pickle_out)
pickle_out.close()
return stars
def compile(self):
h5name_check(self.filename)
hdulist = self.load_allstar()
indices = self.filter_apogeeid_list(hdulist)
info = chips_pix_info(dr=self.apogee_dr)
total_pix = (info[1] - info[0]) + (info[3] - info[2]) + (info[5] -
info[4])
default_length = 900000
spec = np.zeros((default_length, total_pix), dtype=np.float32)
spec_err = np.zeros((default_length, total_pix), dtype=np.float32)
RA = np.zeros(default_length, dtype=np.float32)
DEC = np.zeros(default_length, dtype=np.float32)
SNR = np.zeros(default_length, dtype=np.float32)
individual_flag = np.zeros(default_length, dtype=np.float32)
Kmag = np.zeros(default_length, dtype=np.float32)
AK_TARG = np.zeros(default_length, dtype=np.float32)
# Data array
teff = np.zeros(default_length, dtype=np.float32)
logg = np.zeros(default_length, dtype=np.float32)
MH = np.zeros(default_length, dtype=np.float32)
alpha_M = np.zeros(default_length, dtype=np.float32)
C = np.zeros(default_length, dtype=np.float32)
C1 = np.zeros(default_length, dtype=np.float32)
N = np.zeros(default_length, dtype=np.float32)
O = np.zeros(default_length, dtype=np.float32)
Na = np.zeros(default_length, dtype=np.float32)
Mg = np.zeros(default_length, dtype=np.float32)
Al = np.zeros(default_length, dtype=np.float32)
Si = np.zeros(default_length, dtype=np.float32)
P = np.zeros(default_length, dtype=np.float32)
S = np.zeros(default_length, dtype=np.float32)
K = np.zeros(default_length, dtype=np.float32)
Ca = np.zeros(default_length, dtype=np.float32)
Ti = np.zeros(default_length, dtype=np.float32)
Ti2 = np.zeros(default_length, dtype=np.float32)
V = np.zeros(default_length, dtype=np.float32)
Cr = np.zeros(default_length, dtype=np.float32)
Mn = np.zeros(default_length, dtype=np.float32)
Fe = np.zeros(default_length, dtype=np.float32)
Co = np.zeros(default_length, dtype=np.float32)
Ni = np.zeros(default_length, dtype=np.float32)
Cu = np.zeros(default_length, dtype=np.float32)
Ge = np.zeros(default_length, dtype=np.float32)
Ce = np.zeros(default_length, dtype=np.float32)
Rb = np.zeros(default_length, dtype=np.float32)
Y = np.zeros(default_length, dtype=np.float32)
Nd = np.zeros(default_length, dtype=np.float32)
parallax = np.zeros(default_length, dtype=np.float32)
fakemag = np.zeros(default_length, dtype=np.float32)
# Error array
teff_err = np.zeros(default_length, dtype=np.float32)
logg_err = np.zeros(default_length, dtype=np.float32)
MH_err = np.zeros(default_length, dtype=np.float32)
alpha_M_err = np.zeros(default_length, dtype=np.float32)
C_err = np.zeros(default_length, dtype=np.float32)
C1_err = np.zeros(default_length, dtype=np.float32)
N_err = np.zeros(default_length, dtype=np.float32)
O_err = np.zeros(default_length, dtype=np.float32)
Na_err = np.zeros(default_length, dtype=np.float32)
Mg_err = np.zeros(default_length, dtype=np.float32)
Al_err = np.zeros(default_length, dtype=np.float32)
Si_err = np.zeros(default_length, dtype=np.float32)
P_err = np.zeros(default_length, dtype=np.float32)
S_err = np.zeros(default_length, dtype=np.float32)
K_err = np.zeros(default_length, dtype=np.float32)
Ca_err = np.zeros(default_length, dtype=np.float32)
Ti_err = np.zeros(default_length, dtype=np.float32)
Ti2_err = np.zeros(default_length, dtype=np.float32)
V_err = np.zeros(default_length, dtype=np.float32)
Cr_err = np.zeros(default_length, dtype=np.float32)
Mn_err = np.zeros(default_length, dtype=np.float32)
Fe_err = np.zeros(default_length, dtype=np.float32)
Co_err = np.zeros(default_length, dtype=np.float32)
Ni_err = np.zeros(default_length, dtype=np.float32)
Cu_err = np.zeros(default_length, dtype=np.float32)
Ge_err = np.zeros(default_length, dtype=np.float32)
Ce_err = np.zeros(default_length, dtype=np.float32)
Rb_err = np.zeros(default_length, dtype=np.float32)
Y_err = np.zeros(default_length, dtype=np.float32)
Nd_err = np.zeros(default_length, dtype=np.float32)
parallax_err = np.zeros(default_length, dtype=np.float32)
fakemag_err = np.zeros(default_length, dtype=np.float32)
array_counter = 0
start_time = time.time()
# provide a cont mask so no need to read every loop
if self.cont_mask is None:
maskpath = os.path.join(astroNN.data.datapath(),
f'dr{self.apogee_dr}_contmask.npy')
self.cont_mask = np.load(maskpath)
for counter, index in enumerate(indices):
nvisits = 1
apogee_id = hdulist[1].data['APOGEE_ID'][index]
location_id = hdulist[1].data['LOCATION_ID'][index]
if counter % 100 == 0:
print(
f'Completed {counter + 1} of {indices.shape[0]}, {(time.time() - start_time):.{2}f}s elapsed'
)
if not self.continuum:
path = combined_spectra(dr=self.apogee_dr,
location=location_id,
apogee=apogee_id,
verbose=0)
if path is False:
# if path is not found then we should skip
continue
combined_file = fits.open(path)
_spec = combined_file[
1].data # Pseudo-continuum normalized flux
_spec_err = combined_file[2].data # Spectrum error array
_spec = gap_delete(
_spec, dr=self.apogee_dr) # Delete the gap between sensors
_spec_err = gap_delete(_spec_err, dr=self.apogee_dr)
inSNR = combined_file[0].header['SNR']
combined_file.close()
else:
path = visit_spectra(dr=self.apogee_dr,
location=location_id,
apogee=apogee_id,
verbose=0)
if path is False:
# if path is not found then we should skip
continue
apstar_file = fits.open(path)
nvisits = apstar_file[0].header['NVISITS']
if nvisits == 1:
_spec = apstar_file[1].data
_spec_err = apstar_file[2].data
_spec_mask = apstar_file[3].data
inSNR = np.ones(nvisits)
inSNR[0] = apstar_file[0].header['SNR']
else:
_spec = apstar_file[1].data[1:]
_spec_err = apstar_file[2].data[1:]
_spec_mask = apstar_file[3].data[1:]
inSNR = np.ones(nvisits + 1)
inSNR[0] = apstar_file[0].header['SNR']
for i in range(nvisits):
inSNR[i + 1] = apstar_file[0].header[f'SNRVIS{i + 1}']
# Deal with spectra thats all zeros flux
ii = 0
while ii < _spec.shape[0]:
if np.count_nonzero(_spec[ii]) == 0:
nvisits -= 1
_spec = np.delete(_spec, ii, 0)
_spec_err = np.delete(_spec_err, ii, 0)
_spec_mask = np.delete(_spec_mask, ii, 0)
inSNR = np.delete(inSNR, ii, 0)
ii -= 1
ii += 1
# Just for the sake of program to work, the real nvisits still nvisits
nvisits += 1
# Normalize spectra and Set some bitmask to 0
_spec, _spec_err = self.apstar_normalization(
_spec, _spec_err, _spec_mask)
apstar_file.close()
if nvisits == 1:
individual_flag[array_counter:array_counter + nvisits] = 0
else:
individual_flag[array_counter:array_counter + 1] = 0
individual_flag[array_counter + 1:array_counter + nvisits] = 1
spec[array_counter:array_counter + nvisits, :] = _spec
spec_err[array_counter:array_counter + nvisits, :] = _spec_err
SNR[array_counter:array_counter + nvisits] = inSNR
RA[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['RA'][index], nvisits)
DEC[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['DEC'][index], nvisits)
parallax[array_counter:array_counter + nvisits] = np.tile(
-9999, nvisits)
parallax_err[array_counter:array_counter + nvisits] = np.tile(
-9999, nvisits)
fakemag[array_counter:array_counter + nvisits] = np.tile(
-9999, nvisits)
fakemag_err[array_counter:array_counter + nvisits] = np.tile(
-9999, nvisits)
Kmag[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['K'][index], nvisits)
AK_TARG[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['AK_TARG'][index], nvisits)
if self.spectra_only is not True:
teff[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['PARAM'][index, 0], nvisits)
logg[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['PARAM'][index, 1], nvisits)
MH[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['PARAM'][index, 3], nvisits)
alpha_M[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['PARAM'][index, 6], nvisits)
C[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 0], nvisits)
C1[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 1], nvisits)
N[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 2], nvisits)
O[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 3], nvisits)
Na[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 4], nvisits)
Mg[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 5], nvisits)
Al[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 6], nvisits)
Si[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 7], nvisits)
P[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 8], nvisits)
S[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 9], nvisits)
K[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 10], nvisits)
Ca[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 11], nvisits)
Ti[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 12], nvisits)
Ti2[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 13], nvisits)
V[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 14], nvisits)
Cr[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 15], nvisits)
Mn[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 16], nvisits)
Fe[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 17], nvisits)
Co[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 18], nvisits)
Ni[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 19], nvisits)
Cu[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 20], nvisits)
Ge[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 21], nvisits)
Ce[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 22], nvisits)
Rb[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 23], nvisits)
Y[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 24], nvisits)
Nd[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H'][index, 25], nvisits)
if self.use_err is True:
teff_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['TEFF_ERR'][index], nvisits)
logg_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['LOGG_ERR'][index], nvisits)
MH_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['M_H_ERR'][index], nvisits)
alpha_M_err[array_counter:array_counter +
nvisits] = np.tile(
hdulist[1].data['ALPHA_M_ERR'][index],
nvisits)
C_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 0], nvisits)
C1_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 1], nvisits)
N_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 2], nvisits)
O_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 3], nvisits)
Na_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 4], nvisits)
Mg_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 5], nvisits)
Al_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 6], nvisits)
Si_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 7], nvisits)
P_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 8], nvisits)
S_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 9], nvisits)
K_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 10], nvisits)
Ca_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 11], nvisits)
Ti_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 12], nvisits)
Ti2_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 13], nvisits)
V_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 14], nvisits)
Cr_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 15], nvisits)
Mn_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 16], nvisits)
Fe_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 17], nvisits)
Co_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 18], nvisits)
Ni_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 19], nvisits)
Cu_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 20], nvisits)
Ge_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 21], nvisits)
Ce_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 22], nvisits)
Rb_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 23], nvisits)
Y_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 24], nvisits)
Nd_err[array_counter:array_counter + nvisits] = np.tile(
hdulist[1].data['X_H_ERR'][index, 25], nvisits)
array_counter += nvisits
spec = spec[0:array_counter]
spec_err = spec_err[0:array_counter]
individual_flag = individual_flag[0:array_counter]
RA = RA[0:array_counter]
DEC = DEC[0:array_counter]
SNR = SNR[0:array_counter]
if self.spectra_only is not True:
teff = teff[0:array_counter]
logg = logg[0:array_counter]
Kmag = Kmag[0:array_counter]
AK_TARG = AK_TARG[0:array_counter]
MH = MH[0:array_counter]
alpha_M = alpha_M[0:array_counter]
C = C[0:array_counter]
C1 = C1[0:array_counter]
N = N[0:array_counter]
O = O[0:array_counter]
Na = Na[0:array_counter]
Mg = Mg[0:array_counter]
Al = Al[0:array_counter]
Si = Si[0:array_counter]
P = P[0:array_counter]
S = S[0:array_counter]
K = K[0:array_counter]
Ca = Ca[0:array_counter]
Ti = Ti[0:array_counter]
Ti2 = Ti2[0:array_counter]
V = V[0:array_counter]
Cr = Cr[0:array_counter]
Mn = Mn[0:array_counter]
Fe = Fe[0:array_counter]
Co = Co[0:array_counter]
Ni = Ni[0:array_counter]
Cu = Cu[0:array_counter]
Ge = Ge[0:array_counter]
Ce = Ce[0:array_counter]
Rb = Rb[0:array_counter]
Y = Y[0:array_counter]
Nd = Nd[0:array_counter]
parallax = parallax[0:array_counter]
fakemag = fakemag[0:array_counter]
teff_err = teff_err[0:array_counter]
logg_err = logg_err[0:array_counter]
MH_err = MH_err[0:array_counter]
alpha_M_err = alpha_M_err[0:array_counter]
C_err = C_err[0:array_counter]
C1_err = C1_err[0:array_counter]
N_err = N_err[0:array_counter]
O_err = O_err[0:array_counter]
Na_err = Na_err[0:array_counter]
Mg_err = Mg_err[0:array_counter]
Al_err = Al_err[0:array_counter]
Si_err = Si_err[0:array_counter]
P_err = P_err[0:array_counter]
S_err = S_err[0:array_counter]
K_err = K_err[0:array_counter]
Ca_err = Ca_err[0:array_counter]
Ti_err = Ti_err[0:array_counter]
Ti2_err = Ti2_err[0:array_counter]
V_err = V_err[0:array_counter]
Cr_err = Cr_err[0:array_counter]
Mn_err = Mn_err[0:array_counter]
Fe_err = Fe_err[0:array_counter]
Co_err = Co_err[0:array_counter]
Ni_err = Ni_err[0:array_counter]
Cu_err = Cu_err[0:array_counter]
Ge_err = Ge_err[0:array_counter]
Ce_err = Ce_err[0:array_counter]
Rb_err = Rb_err[0:array_counter]
Y_err = Y_err[0:array_counter]
Nd_err = Nd_err[0:array_counter]
parallax_err = parallax_err[0:array_counter]
fakemag_err = fakemag_err[0:array_counter]
if self.use_esa_gaia is True:
gaia_ra, gaia_dec, gaia_parallax, gaia_err = gaiadr2_parallax(
cuts=True, keepdims=False)
m1, m2, sep = xmatch(RA,
gaia_ra,
maxdist=2,
colRA1=RA,
colDec1=DEC,
colRA2=gaia_ra,
colDec2=gaia_dec,
swap=False)
parallax[m1] = gaia_parallax[m2]
parallax_err[m1] = gaia_err[m2]
fakemag[m1], fakemag_err[m1] = mag_to_fakemag(
extinction_correction(Kmag[m1], AK_TARG[m1]), parallax[m1],
parallax_err[m1])
elif self.use_anderson_2017 is True:
gaia_ra, gaia_dec, gaia_parallax, gaia_err = anderson_2017_parallax(
)
m1, m2, sep = xmatch(RA,
gaia_ra,
maxdist=2,
colRA1=RA,
colDec1=DEC,
epoch1=2000.,
colRA2=gaia_ra,
colDec2=gaia_dec,
epoch2=2000.,
swap=False)
parallax[m1] = gaia_parallax[m2]
parallax_err[m1] = gaia_err[m2]
fakemag[m1], fakemag_err[m1] = mag_to_fakemag(
extinction_correction(Kmag[m1], AK_TARG[m1]), parallax[m1],
parallax_err[m1])
print(f'Creating {self.filename}.h5')
h5f = h5py.File(f'{self.filename}.h5', 'w')
h5f.create_dataset('spectra', data=spec)
h5f.create_dataset('spectra_err', data=spec_err)
h5f.create_dataset('in_flag', data=individual_flag)
h5f.create_dataset('index', data=indices)
if self.spectra_only is not True:
h5f.create_dataset('SNR', data=SNR)
h5f.create_dataset('RA', data=RA)
h5f.create_dataset('DEC', data=DEC)
h5f.create_dataset('Kmag', data=Kmag)
h5f.create_dataset('AK_TARG', data=AK_TARG)
h5f.create_dataset('teff', data=teff)
h5f.create_dataset('logg', data=logg)
h5f.create_dataset('M', data=MH)
h5f.create_dataset('alpha', data=alpha_M)
h5f.create_dataset('C', data=C)
h5f.create_dataset('C1', data=C1)
h5f.create_dataset('N', data=N)
h5f.create_dataset('O', data=O)
h5f.create_dataset('Na', data=Na)
h5f.create_dataset('Mg', data=Mg)
h5f.create_dataset('Al', data=Al)
h5f.create_dataset('Si', data=Si)
h5f.create_dataset('P', data=P)
h5f.create_dataset('S', data=S)
h5f.create_dataset('K', data=K)
h5f.create_dataset('Ca', data=Ca)
h5f.create_dataset('Ti', data=Ti)
h5f.create_dataset('Ti2', data=Ti2)
h5f.create_dataset('V', data=V)
h5f.create_dataset('Cr', data=Cr)
h5f.create_dataset('Mn', data=Mn)
h5f.create_dataset('Fe', data=Fe)
h5f.create_dataset('Co', data=Co)
h5f.create_dataset('Ni', data=Ni)
h5f.create_dataset('Cu', data=Cu)
h5f.create_dataset('Ge', data=Ge)
h5f.create_dataset('Ce', data=Ce)
h5f.create_dataset('Rb', data=Rb)
h5f.create_dataset('Y', data=Y)
h5f.create_dataset('Nd', data=Nd)
h5f.create_dataset('parallax', data=parallax)
h5f.create_dataset('fakemag', data=fakemag)
if self.use_err is True:
h5f.create_dataset('AK_TARG_err', data=np.zeros_like(AK_TARG))
h5f.create_dataset('teff_err', data=teff_err)
h5f.create_dataset('logg_err', data=logg_err)
h5f.create_dataset('M_err', data=MH_err)
h5f.create_dataset('alpha_err', data=alpha_M_err)
h5f.create_dataset('C_err', data=C_err)
h5f.create_dataset('C1_err', data=C1_err)
h5f.create_dataset('N_err', data=N_err)
h5f.create_dataset('O_err', data=O_err)
h5f.create_dataset('Na_err', data=Na_err)
h5f.create_dataset('Mg_err', data=Mg_err)
h5f.create_dataset('Al_err', data=Al_err)
h5f.create_dataset('Si_err', data=Si_err)
h5f.create_dataset('P_err', data=P_err)
h5f.create_dataset('S_err', data=S_err)
h5f.create_dataset('K_err', data=K_err)
h5f.create_dataset('Ca_err', data=Ca_err)
h5f.create_dataset('Ti_err', data=Ti_err)
h5f.create_dataset('Ti2_err', data=Ti2_err)
h5f.create_dataset('V_err', data=V_err)
h5f.create_dataset('Cr_err', data=Cr_err)
h5f.create_dataset('Mn_err', data=Mn_err)
h5f.create_dataset('Fe_err', data=Fe_err)
h5f.create_dataset('Co_err', data=Co_err)
h5f.create_dataset('Ni_err', data=Ni_err)
h5f.create_dataset('Cu_err', data=Cu_err)
h5f.create_dataset('Ge_err', data=Ge_err)
h5f.create_dataset('Ce_err', data=Ce_err)
h5f.create_dataset('Rb_err', data=Rb_err)
h5f.create_dataset('Y_err', data=Y_err)
h5f.create_dataset('Nd_err', data=Nd_err)
h5f.create_dataset('parallax_err', data=parallax_err)
h5f.create_dataset('fakemag_err', data=fakemag_err)
h5f.close()
print(f'Successfully created {self.filename}.h5 in {currentdir}')