def chips_split(spectra, dr=None):
"""
To split APOGEE spectra into RGB chips, will delete the gap if detected
:param spectra: APOGEE spectrum/spectra
:type spectra: ndarray
:param dr: data release
:type dr: Union(int, NoneType)
:return: 3 ndarrays which are spectra_blue, spectra_green, spectra_red
:rtype: ndarray
:History:
| 2017-Nov-20 - Written - Henry Leung (University of Toronto)
| 2017-Dec-17 - Updated - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
info = chips_pix_info(dr=dr)
blue = info[1] - info[0]
green = info[3] - info[2]
red = info[5] - info[4]
spectra = np.atleast_2d(spectra)
if spectra.shape[1] == 8575:
spectra = gap_delete(spectra, dr=dr)
print("Raw Spectra detected, astroNN has deleted the gap automatically")
elif spectra.shape[1] == info[6]:
pass
else:
raise EnvironmentError('Are you sure you are giving astroNN APOGEE spectra?')
spectra_blue = spectra[:, 0:blue]
spectra_green = spectra[:, blue:(blue + green)]
spectra_red = spectra[:, (blue + green):(blue + green + red)]
return spectra_blue, spectra_green, spectra_red
def gap_delete(spectra, dr=None):
"""
To delete the gap between APOGEE CCDs from the original 8575 pixels spectra
:param spectra: The original 8575 pixels spectrum/spectra
:type spectra: ndarray
:param dr: data release
:type dr: Union(int, NoneType)
:return: Gap deleted spectrum/spectra
:rtype: ndarray
:History:
| 2017-Oct-26 - Written - Henry Leung (University of Toronto)
| 2017-Dec-16 - Updated - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
spectra = np.atleast_2d(spectra)
info = chips_pix_info(dr=dr)
if spectra.shape[1] != 8575 and spectra.shape[1] != info[6]:
raise EnvironmentError(
'Are you sure you are giving astroNN APOGEE spectra?')
if spectra.shape[1] != info[6]:
spectra = spectra[:, np.r_[info[0]:info[1], info[2]:info[3],
info[4]:info[5]]]
return spectra
def wavelength_solution(dr=None):
"""
To return wavelegnth_solution, apStarWavegrid was provided by Jo Bovy's apogee tools (Toronto)
:param dr: data release
:type dr: Union(int, NoneType)
:return:
| lambda_blue, lambda_green, lambda_red which are 3 wavelength solution array
| - lambda_blue refers to the wavelength solution for each pixel in blue chips
| - lambda_green refers to the wavelength solution for each pixel in green chips
| - lambda_red refers to the wavelength solution for each pixel in red chips
:rtype: ndarray
:History:
| 2017-Nov-20 - Written - Henry Leung (University of Toronto)
| 2017-Dec-16 - Updated - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
info = chips_pix_info(dr=dr)
apstar_wavegrid = 10.**np.arange(4.179, 4.179 + 8575 * 6. * 10.**-6.,
6. * 10.**-6.)
lambda_blue = apstar_wavegrid[info[0]:info[1]]
lambda_green = apstar_wavegrid[info[2]:info[3]]
lambda_red = apstar_wavegrid[info[4]:info[5]]
return lambda_blue, lambda_green, lambda_red
def combined_spectra(dr=None, location=None, apogee=None, verbose=1):
"""
NAME: combined_spectra
PURPOSE: download the required combined spectra file (catalog of properties from individual visit spectra)
INPUT: Data Release 13 OR 14
OUTPUT: (just downloads)
HISTORY:
2017-Oct-15 Henry Leung
"""
warning_flag = None
dr = apogee_default_dr(dr=dr)
if dr == 13:
str1 = 'https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/'
str2 = '{}/aspcapStar-r6-l30e.2-{}.fits'.format(location, apogee)
filename = 'aspcapStar-r6-l30e.2-{}.fits'.format(apogee)
urlstr = str1 + str2
fullfilename = os.path.join(_APOGEE_DATA, 'dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/', str(location))
if not os.path.exists(fullfilename):
os.makedirs(fullfilename)
fullfilename = os.path.join(_APOGEE_DATA, 'dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/', str(location),
filename)
if not os.path.isfile(fullfilename):
try:
urllib.request.urlretrieve(urlstr, fullfilename)
print('Downloaded DR13 combined file successfully to {}'.format(fullfilename))
except urllib.request.HTTPError:
print('{} cannot be found on server, skipped'.format(urlstr))
else:
print(fullfilename + ' was found, not downloaded again')
elif dr == 14:
str1 = 'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/'
str2 = '{}/aspcapStar-r8-l31c.2-{}.fits'.format(location, apogee)
filename = 'aspcapStar-r8-l31c.2-{}.fits'.format(apogee)
urlstr = str1 + str2
fullfilename = os.path.join(_APOGEE_DATA, 'dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/', str(location))
if not os.path.exists(fullfilename):
os.makedirs(fullfilename)
fullfilename = os.path.join(_APOGEE_DATA, 'dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/', str(location),
filename)
if not os.path.isfile(fullfilename):
try:
urllib.request.urlretrieve(urlstr, fullfilename)
print('Downloaded DR14 combined file successfully to {}'.format(fullfilename))
except urllib.request.HTTPError:
print('{} cannot be found on server, skipped'.format(urlstr))
warning_flag = 1
else:
if verbose == 1:
print(fullfilename + ' was found, not downloaded again')
else:
raise ValueError('combined_spectra() only supports DR13 or DR14')
return warning_flag, fullfilename
def allvisit(dr=None, flag=None):
"""
Download the allVisit file (catalog of properties from individual visit spectra)
:param dr: APOGEE DR
:type dr: int
:param flag: 0: normal, 1: force to re-download
:type flag: int
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History: 2017-Oct-11 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 13:
file_hash = '2a3b13ccd40a2c8aea8321be9630117922d55b51'
# Check if directory exists
fullfilepath = os.path.join(apogee_env(), 'dr13/apogee/spectro/redux/r6/')
if not os.path.exists(fullfilepath):
os.makedirs(fullfilepath)
filename = 'allVisit-l30e.2.fits'
fullfilename = os.path.join(fullfilepath, filename)
url = f'https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/{filename}'
elif dr == 14:
file_hash = 'abcecbcdc5fe8d00779738702c115633811e6bbd'
# Check if directory exists
fullfilepath = os.path.join(apogee_env(), 'dr14/apogee/spectro/redux/r8/')
if not os.path.exists(fullfilepath):
os.makedirs(fullfilepath)
filename = 'allVisit-l31c.2.fits'
fullfilename = os.path.join(fullfilepath, filename)
url = f'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/{filename}'
else:
raise ValueError('allvisit() only supports APOGEE DR13-DR15')
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash.lower():
print('File corruption detected, astroNN is attempting to download again')
allvisit(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
elif not os.path.isfile(os.path.join(fullfilepath, filename)) or flag == 1:
with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:
urllib.request.urlretrieve(url, fullfilename, reporthook=t.update_to)
print(f'Downloaded DR{dr:d} allVisit file catalog successfully to {fullfilepath}')
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash.lower():
print('File corruption detected, astroNN is attempting to download again')
allstar(dr=dr, flag=1)
return fullfilename
def apogee_distances(dr=None, flag=None):
"""
Download the Apogee Distances catalogue
:param dr: Apogee DR
:type dr: int
:param flag: Force to download if flag=1
:type flag: int
:return: full file path
:rtype: str
:History: 2018-Jan-24 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 14:
file_hash = 'b33c8419be784b1be3d14af3ee9696c6ac31830f'
str1 = 'https://data.sdss.org/sas/dr14/apogee/vac/apogee-distances/'
filename = f'apogee_distances-DR{dr}.fits'
urlstr = str1 + filename
fullfoldername = os.path.join(apogee_env(), 'dr14/apogee/vac/apogee-distances/')
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
else:
raise ValueError('apogee_distances() only supports DR14')
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash.lower():
print('File corruption detected, astroNN is attempting to download again')
apogee_distances(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
elif not os.path.isfile(fullfilename) or flag == 1:
try:
with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=urlstr.split('/')[-1]) as t:
urllib.request.urlretrieve(urlstr, fullfilename, reporthook=t.update_to)
print(f'Downloaded DR{dr} Distances successfully to {fullfilename}')
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash.lower():
print('File corruption detected, astroNN is attempting to download again')
apogee_distances(dr=dr, flag=1)
except urllib.error.HTTPError:
print(f'{urlstr} cannot be found on server, skipped')
fullfilename = warning_flag
return fullfilename
def apogee_astronn(dr=None, flag=None):
"""
Download the apogee_astroNN file (catalog of astroNN stellar parameters, abundances, distances and orbital
parameters from combined spectra)
:param dr: APOGEE DR
:type dr: int
:param flag: 0: normal, 1: force to re-download
:type flag: int
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History: 2019-Dec-10 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 16:
# Check if directory exists
fullfoldername = os.path.join(apogee_env(), 'dr16/apogee/vac/apogee-astronn/')
# Check if directory exists
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
filename = 'apogee_astroNN-DR16.fits'
fullfilename = os.path.join(fullfoldername, filename)
file_hash = '02187ef2cbe5215dc4d65df7037ecf1b8cc5853d'
url = f'https://data.sdss.org/sas/dr16/apogee/vac/apogee-astronn/{filename}'
else:
raise ValueError('apogee_astroNN() only supports APOGEE DR16')
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash.lower():
print('File corruption detected, astroNN is attempting to download again')
apogee_astronn(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
# Check if files exists
if not os.path.isfile(os.path.join(fullfoldername, filename)) or flag == 1:
with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:
urllib.request.urlretrieve(url, fullfilename, reporthook=t.update_to)
print(f'Downloaded DR{dr:d} apogee_astroNN file catalog successfully to {fullfilename}')
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash.lower():
print('File corruption detected, astroNN is attempting to download again')
apogee_astronn(dr=dr, flag=1)
return fullfilename
def allstar_cannon(dr=None, flag=None):
"""
Download the allStarCannon file (catalog of Cannon stellar parameters and abundances from combined spectra)
:param dr: APOGEE DR
:type dr: int
:param flag: 0: normal, 1: force to re-download
:type flag: int
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History: 2017-Oct-24 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 14:
# Check if directory exists
fullfoldername = os.path.join(apogee_env(), 'dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/cannon/')
# Check if directory exists
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
filename = 'allStarCannon-l31c.2.fits'
fullfilename = os.path.join(fullfoldername, filename)
file_hash = '64d485e95b3504df0b795ab604e21a71d5c7ae45'
url = f'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/cannon/{filename}'
else:
raise ValueError('allstar_cannon() only supports APOGEE DR14-DR15')
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash.lower():
print('File corruption detected, astroNN is attempting to download again')
allstar_cannon(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
# Check if files exists
if not os.path.isfile(os.path.join(fullfoldername, filename)) or flag == 1:
with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:
urllib.request.urlretrieve(url, fullfilename, reporthook=t.update_to)
print(f'Downloaded DR{dr:d} allStarCannon file catalog successfully to {fullfilename}')
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash.lower():
print('File corruption detected, astroNN is attempting to download again')
allstar_cannon(dr=dr, flag=1)
return fullfilename
def aspcap_mask(elem, dr=None):
"""
| To load ASPCAP elements window masks
| DR14 Elements: ``'C', 'CI', 'N', 'O', 'Na', 'Mg', 'Al', 'Si', 'P', 'S', 'K', 'Ca', 'TI', 'TiII', 'V', 'Cr', 'Mn',
'Fe', 'Co', 'Ni', 'Cu', 'Ge', 'Ce', 'Rb', 'Y', 'Nd'``
:param elem: element name
:type elem: str
:param dr: apogee dr
:type dr: int
:return: mask
:rtype: ndarray[bool]
:History: 2018-Mar-24 - Written - Henry Leung (University of Toronto)
"""
if elem.lower() == 'c1':
elem = 'CI'
elif elem.lower() == 'ti2':
elem = 'TiII'
dr = apogee_default_dr(dr=dr)
if dr == 14:
aspcap_code = 'l31c'
elem_list = [
'C', 'CI', 'N', 'O', 'Na', 'Mg', 'Al', 'Si', 'P', 'S', 'K', 'Ca',
'TI', 'TiII', 'V', 'Cr', 'Mn', 'Fe', 'Co', 'Ni', 'Cu', 'Ge', 'Ce',
'Rb', 'Y', 'Nd'
]
else:
raise ValueError('Only DR14 is supported currently')
masks = np.load(
os.path.join(os.path.dirname(astroNN.__path__[0]), 'astroNN', 'data',
f'aspcap_{aspcap_code}_masks.npy'))
try:
# turn everything to lowercase to avoid case-related issue
index = [x.lower() for x in elem_list].index(elem.lower())
except ValueError:
# nicely handle if element not found
print(
f'Element not found, the only elements for dr{dr} supported are {elem_list}'
)
return None
return [(masks & 2**index) != 0][0]
def chips_pix_info(dr=None):
"""
To return chips info according to dr
:param dr: data release
:type dr: Union(int, NoneType)
:return:
| The starting and ending pixels location of APOGEE camera chips in the original 8575 pixels spectra
| - list[0] refers to the location where blue chips starts
| - list[1] refers to the location where blue chips ends
| - list[2] refers to the location where green chips starts
| - list[3] refers to the location where blue chips end
| - list[4] refers to the location where red chips starts
| - list[5] refers to the location where red chips ends
| - list[6] refers to the total number of pixels after deleting gap
:rtype: list
:History:
| 2017-Nov-27 - Written - Henry Leung (University of Toronto)
| 2017-Dec-16 - Updated - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 11 or dr == 12:
blue_start = 322
blue_end = 3242
green_start = 3648
green_end = 6048
red_start = 6412
red_end = 8306
total_pixel = 7214
elif dr == 13 or dr == 14 or dr == 15:
blue_start = 246
blue_end = 3274
green_start = 3585
green_end = 6080
red_start = 6344
red_end = 8335
total_pixel = 7514
else:
raise ValueError('Only DR11 to DR15 are supported')
return [
blue_start, blue_end, green_start, green_end, red_start, red_end,
total_pixel
]
def allstar(dr=None):
"""
NAME: allstar
PURPOSE: download the allStar file (catalog of ASPCAP stellar parameters and abundances from combined spectra)
INPUT: Data Release 13 OR 14
OUTPUT: full file path and download in background
HISTORY:
2017-Oct-09 Henry Leung
"""
dr = apogee_default_dr(dr=dr)
if dr == 13:
# Check if directory exists
fullfilepath = os.path.join(_APOGEE_DATA, 'dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/')
if not os.path.exists(fullfilepath):
os.makedirs(fullfilepath)
filename = 'allStar-l30e.2.fits'
fullfilename = os.path.join(fullfilepath, filename)
url = 'https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/{}'.format(filename)
elif dr == 14:
fullfilepath = os.path.join(_APOGEE_DATA, 'dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/')
# Check if directory exists
if not os.path.exists(fullfilepath):
os.makedirs(fullfilepath)
filename = 'allStar-l31c.2.fits'
fullfilename = os.path.join(fullfilepath, filename)
url = 'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/{}'.format(filename)
else:
raise ValueError('[astroNN.apogee.downloader.all_star()] only supports APOGEE DR13 and DR14')
# Check if files exists
if not os.path.isfile(os.path.join(fullfilepath, filename)):
with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:
urllib.request.urlretrieve(url, fullfilename, reporthook=t.update_to)
print('Downloaded DR{:d} allStar file catalog successfully to {}'.format(dr, fullfilename))
else:
print(fullfilename + ' was found!')
return fullfilename
def allvisit(dr=None):
"""
NAME: allvisit
PURPOSE: download the allVisit file (catalog of properties from individual visit spectra)
INPUT: Data Release 13 OR 14
OUTPUT: (just downloads)
HISTORY:
2017-Oct-11 Henry Leung
"""
dr = apogee_default_dr(dr=dr)
if dr == 13:
# Check if directory exists
fullfilepath = os.path.join(_APOGEE_DATA, 'dr13/apogee/spectro/redux/r6/')
if not os.path.exists(fullfilepath):
os.makedirs(fullfilepath)
filename = 'allVisit-l30e.2.fits'
fullfilename = os.path.join(fullfilepath, filename)
url = 'https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/{}'.format(filename)
elif dr == 14:
# Check if directory exists
fullfilepath = os.path.join(_APOGEE_DATA, 'dr14/apogee/spectro/redux/r8/')
if not os.path.exists(fullfilepath):
os.makedirs(fullfilepath)
filename = 'allVisit-l31c.2.fits'
fullfilename = os.path.join(fullfilepath, filename)
url = 'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/{}'.format(filename)
else:
raise ValueError('[astroNN.apogee.downloader.all_visit()] only supports APOGEE DR13 and DR14')
if not os.path.isfile(os.path.join(fullfilepath, filename)):
with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:
urllib.request.urlretrieve(url, fullfilename, reporthook=t.update_to)
print('Downloaded DR{:d} allVisit file catalog successfully to {}'.format(dr, fullfilepath))
else:
print(fullfilename + ' was found')
return None
def combined_spectra(dr=None,
location=None,
field=None,
apogee=None,
telescope=None,
verbose=1,
flag=None):
"""
Download the required combined spectra file a.k.a aspcapStar
:param dr: APOGEE DR
:type dr: int
:param location: Location ID [Optional]
:type location: int
:param field: Field [Optional]
:type field: str
:param apogee: Apogee ID
:type apogee: str
:param telescope: Telescope ID, for example 'apo25m' or 'lco25m'
:type telescope: str
:param verbose: verbose, set 0 to silent most logging
:type verbose: int
:param flag: 0: normal, 1: force to re-download
:type flag: int
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History:
| 2017-Oct-15 - Written - Henry Leung (University of Toronto)
| 2018-Aug-31 - Updated - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if location is None and field is None: # for DR16=<, location is expected to be none because field is used
global _ALLSTAR_TEMP
if not str(f'dr{dr}') in _ALLSTAR_TEMP:
_ALLSTAR_TEMP[f'dr{dr}'] = fits.getdata(allstar(dr=dr))
if telescope is None:
matched_idx = [
np.nonzero(_ALLSTAR_TEMP[f'dr{dr}']['APOGEE_ID'] == apogee)[0]
][0]
else:
matched_idx = [
np.nonzero([
(_ALLSTAR_TEMP[f'dr{dr}']['APOGEE_ID'] == apogee) &
(_ALLSTAR_TEMP[f'dr{dr}']['TELESCOPE'] == telescope)
])
][0][1]
if len(matched_idx) == 0:
raise ValueError(
f"No entry found in allstar DR{dr} met with your requirement!!"
)
location = _ALLSTAR_TEMP[f'dr{dr}']['LOCATION_ID'][matched_idx][0]
field = _ALLSTAR_TEMP[f'dr{dr}']['FIELD'][matched_idx][0]
telescope = _ALLSTAR_TEMP[f'dr{dr}']['TELESCOPE'][matched_idx][0]
if dr == 13:
reduce_prefix = 'r6'
aspcap_code = 'l30e'
str1 = f'https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/{location}/'
filename = f'aspcapStar-{reduce_prefix}-{aspcap_code}.2-{apogee}.fits'
hash_filename = f'stars_{aspcap_code}_{aspcap_code}.2_{location}.sha1sum'
urlstr = str1 + filename
# check folder existence
fullfoldername = os.path.join(
apogee_env(),
f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/',
str(location))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
elif dr == 14:
reduce_prefix = 'r8'
aspcap_code = 'l31c'
str1 = f'https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/{location}/'
filename = f'aspcapStar-{reduce_prefix}-{aspcap_code}.2-{apogee}.fits'
hash_filename = f'stars_{aspcap_code}_{aspcap_code}.2_{location}.sha1sum'
urlstr = str1 + filename
# check folder existence
fullfoldername = os.path.join(
apogee_env(),
f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/',
str(location))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
elif dr == 16:
reduce_prefix = 'r12'
aspcap_code = 'l33'
str1 = f'https://data.sdss.org/sas/apogeework/apogee/spectro/aspcap/{reduce_prefix}/{aspcap_code}/{telescope}/{field}/'
filename = f'aspcapStar-{reduce_prefix}-{apogee}.fits'
hash_filename = f'stars_{reduce_prefix}_{reduce_prefix}.2_{location}.sha1sum'
urlstr = str1 + filename
# check folder existence
fullfoldername = os.path.join(
apogee_env(),
f'dr{dr}/apogee/spectro/aspcap/{reduce_prefix}/{aspcap_code}/{telescope}',
str(f'{field}'))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
else:
raise ValueError('combined_spectra() only supports DR13-DR16')
# check hash file
if dr != 16:
full_hash_filename = os.path.join(fullfoldername, hash_filename)
if not os.path.isfile(full_hash_filename):
# return warning flag if the location_id cannot even be found
try:
urllib.request.urlopen(str1)
except urllib.request.HTTPError:
return warning_flag
urllib.request.urlretrieve(str1 + hash_filename,
full_hash_filename)
hash_list = np.loadtxt(full_hash_filename, dtype='str').T
else:
# just a dummy list
hash_list = np.array(
[np.array(['yyy', 'yyy', 'yyy']),
np.array(['zzz', 'zzz', 'zzz'])])
# In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too
file_hash = hash_list[0][np.argwhere(hash_list[1] == filename)]
if os.path.isfile(fullfilename) and flag is None:
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash and len(file_hash) != 0:
print(
'File corruption detected, astroNN attempting to download again'
)
combined_spectra(dr=dr,
location=location,
apogee=apogee,
verbose=verbose,
flag=1)
if verbose == 1:
print(fullfilename + ' was found!')
elif not os.path.isfile(fullfilename) or flag == 1:
try:
urllib.request.urlretrieve(urlstr, fullfilename)
print(
f'Downloaded DR{dr} combined file successfully to {fullfilename}'
)
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash and len(file_hash) != 0:
print(
'File corruption detected, astroNN attempting to download again'
)
combined_spectra(dr=dr,
location=location,
apogee=apogee,
verbose=verbose,
flag=1)
except urllib.request.HTTPError as emsg:
if '401' in str(emsg):
fullfilename = __apogee_credentials_downloader(
urlstr, fullfilename)
elif '404' in str(emsg):
print(f'{urlstr} cannot be found on server, skipped')
fullfilename = warning_flag
else:
print(f"Unknown error occurred - {emsg}")
fullfilename = warning_flag
return fullfilename
def apogee_distances(dr=None, flag=None):
"""
Download the APOGEE Distances VAC catalogue (APOGEE Distances for DR14, APOGEE Starhourse for DR16)
:param dr: APOGEE DR
:type dr: int
:param flag: Force to download if flag=1
:type flag: int
:return: full file path
:rtype: str
:History:
| 2018-Jan-24 - Written - Henry Leung (University of Toronto)
| 2021-Jan-29 - Updated - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 14:
file_hash = "b33c8419be784b1be3d14af3ee9696c6ac31830f"
str1 = "https://data.sdss.org/sas/dr14/apogee/vac/apogee-distances/"
filename = f"apogee_distances-DR{dr}.fits"
urlstr = str1 + filename
fullfoldername = os.path.join(apogee_env(),
"dr14/apogee/vac/apogee-distances/")
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
if dr == 16:
file_hash = "2502e2f7703046163f81ecc4054dce39b2038e4f"
str1 = "https://data.sdss.org/sas/dr16/apogee/vac/apogee-starhorse/"
filename = f"apogee_starhorse-DR{dr}-v1.fits"
urlstr = str1 + filename
fullfoldername = os.path.join(apogee_env(),
"dr16/apogee/vac/apogee-starhorse/")
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
else:
raise ValueError("apogee_distances() only supports APOGEE DR14-DR16")
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = filehash(fullfilename, algorithm="sha1")
if checksum != file_hash.lower():
warnings.warn(
"File corruption detected, astroNN is attempting to download again"
)
apogee_distances(dr=dr, flag=1)
else:
logging.info(fullfilename + " was found!")
elif not os.path.isfile(fullfilename) or flag == 1:
try:
with TqdmUpTo(unit="B",
unit_scale=True,
miniters=1,
desc=urlstr.split("/")[-1]) as t:
urllib.request.urlretrieve(urlstr,
fullfilename,
reporthook=t.update_to)
logging.info(
f"Downloaded DR{dr} Distances successfully to {fullfilename}"
)
checksum = filehash(fullfilename, algorithm="sha1")
if checksum != file_hash.lower():
warnings.warn(
"File corruption detected, astroNN is attempting to download again"
)
apogee_distances(dr=dr, flag=1)
except urllib.error.HTTPError:
warnings.warn(f"{urlstr} cannot be found on server, skipped")
fullfilename = warning_flag
return fullfilename
def apogee_vac_rc(dr=None, flag=None):
"""
Download the red clumps catalogue
:param dr: Apogee DR
:type dr: int
:param flag: Force to download if flag=1
:type flag: int
:return: full file path
:rtype: str
:History: 2017-Nov-16 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 13:
file_hash = '5e87eb3ba202f9db24216978dafb19d39d382fc6'
str1 = 'https://data.sdss.org/sas/dr13/apogee/vac/apogee-rc/cat/'
filename = f'apogee-rc-DR{dr}.fits'
urlstr = str1 + filename
fullfoldername = os.path.join(apogee_env(), 'dr13/apogee/vac/apogee-rc/cat/')
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
elif dr == 14:
file_hash = '104513070f1c280954f3d1886cac429dbdf2eaf6'
str1 = 'https://data.sdss.org/sas/dr14/apogee/vac/apogee-rc/cat/'
filename = f'apogee-rc-DR{dr}.fits'
urlstr = str1 + filename
fullfoldername = os.path.join(apogee_env(), 'dr14/apogee/vac/apogee-rc/cat/')
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
elif dr == 16:
file_hash = '0bc75a230058f50ed8a5ea3fa8554d803ffc103d'
str1 = 'https://data.sdss.org/sas/dr16/apogee/vac/apogee-rc/cat/'
filename = f'apogee-rc-DR{dr}.fits'
urlstr = str1 + filename
fullfoldername = os.path.join(apogee_env(), 'dr16/apogee/vac/apogee-rc/cat/')
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
else:
raise ValueError('apogee_vac_rc() only supports DR13 or DR14')
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash.lower():
print('File corruption detected, astroNN is attempting to download again')
apogee_vac_rc(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
elif not os.path.isfile(fullfilename) or flag == 1:
try:
with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=urlstr.split('/')[-1]) as t:
urllib.request.urlretrieve(urlstr, fullfilename, reporthook=t.update_to)
print(f'Downloaded DR{dr} Red Clumps Catalog successfully to {fullfilename}')
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash.lower():
print('File corruption detected, astroNN is attempting to download again')
apogee_vac_rc(dr=dr, flag=1)
except urllib.error.HTTPError:
print(f'{urlstr} cannot be found on server, skipped')
fullfilename = warning_flag
return fullfilename
def allvisit(dr=None, flag=None):
"""
Download the allVisit file (catalog of properties from individual visit spectra)
:param dr: APOGEE DR
:type dr: int
:param flag: 0: normal, 1: force to re-download
:type flag: int
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History: 2017-Oct-11 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 13:
file_hash = "2a3b13ccd40a2c8aea8321be9630117922d55b51"
# Check if directory exists
fullfilepath = os.path.join(apogee_env(),
"dr13/apogee/spectro/redux/r6/")
if not os.path.exists(fullfilepath):
os.makedirs(fullfilepath)
filename = "allVisit-l30e.2.fits"
fullfilename = os.path.join(fullfilepath, filename)
url = f"https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/{filename}"
elif dr == 14:
file_hash = "abcecbcdc5fe8d00779738702c115633811e6bbd"
# Check if directory exists
fullfilepath = os.path.join(apogee_env(),
"dr14/apogee/spectro/redux/r8/")
if not os.path.exists(fullfilepath):
os.makedirs(fullfilepath)
filename = "allVisit-l31c.2.fits"
fullfilename = os.path.join(fullfilepath, filename)
url = f"https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/{filename}"
elif dr == 16:
file_hash = "65befb967d8d9d6f4f87711c1fa8d0ac014b62da"
# Check if directory exists
fullfilepath = os.path.join(apogee_env(),
"dr16/apogee/spectro/aspcap/r12/l33/")
if not os.path.exists(fullfilepath):
os.makedirs(fullfilepath)
filename = "allVisit-r12-l33.fits"
fullfilename = os.path.join(fullfilepath, filename)
url = f"https://data.sdss.org/sas/dr16/apogee/spectro/aspcap/r12/l33/{filename}"
else:
raise ValueError("allvisit() only supports APOGEE DR13-DR16")
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = filehash(fullfilename, algorithm="sha1")
if checksum != file_hash.lower():
warnings.warn(
"File corruption detected, astroNN is attempting to download again"
)
allvisit(dr=dr, flag=1)
else:
logging.info(fullfilename + " was found!")
elif not os.path.isfile(os.path.join(fullfilepath, filename)) or flag == 1:
with TqdmUpTo(unit="B",
unit_scale=True,
miniters=1,
desc=url.split("/")[-1]) as t:
urllib.request.urlretrieve(url,
fullfilename,
reporthook=t.update_to)
logging.info(
f"Downloaded DR{dr:d} allVisit file catalog successfully to {fullfilepath}"
)
checksum = filehash(fullfilename, algorithm="sha1")
if checksum != file_hash.lower():
warnings.warn(
"File corruption detected, astroNN is attempting to download again"
)
allvisit(dr=dr, flag=1)
return fullfilename
def apogee_continuum(spectra,
spectra_err,
cont_mask=None,
deg=2,
dr=None,
bitmask=None,
target_bit=None,
mask_value=1.):
"""
It is designed only for apogee spectra by fitting Chebyshev polynomials to the flux values in the continuum mask
by chips. The resulting continuum will have the same shape as `fluxes`.
:param spectra: spectra
:type spectra: ndarray
:param spectra_err: spectra uncertainty, same shape as spectra
:type spectra_err: ndarray
:param cont_mask: continuum mask
:type cont_mask: ndarray[bool]
:param deg: The degree of Chebyshev polynomial to use in each region, default is 2 which works the best so far
:type deg: int
:param dr: apogee dr
:type dr: int
:param bitmask: bitmask array of the spectra, same shape as spectra
:type bitmask: ndarray
:param target_bit: a list of bit to be masked
:type target_bit: Union(int, list[int], ndarray[int])
:param mask_value: if a pixel is determined to be a bad pixel, this value will be used to replace that pixel flux
:type mask_value: Union(int, float)
:return: normalized spectra, normalized spectra uncertainty
:rtype: ndarray, ndarray
:History: 2018-Mar-21 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
spectra = gap_delete(spectra, dr=dr)
flux_errs = gap_delete(spectra_err, dr=dr)
spectra_blue, spectra_green, spectra_red = chips_split(spectra, dr=dr)
yerrs_blue, yerrs_green, yerrs_red = chips_split(flux_errs, dr=dr)
if cont_mask is None:
maskpath = os.path.join(os.path.dirname(astroNN.__path__[0]),
'astroNN', 'data', f'dr{dr}_contmask.npy')
cont_mask = np.load(maskpath)
con_mask_blue, con_mask_green, con_mask_red = chips_split(cont_mask, dr=dr)
con_mask_blue, con_mask_green, con_mask_red = con_mask_blue[
0], con_mask_green[0], con_mask_red[0]
# Continuum chips by chips
blue_spectra, blue_spectra_err = continuum(spectra_blue,
yerrs_blue,
cont_mask=con_mask_blue,
deg=deg)
green_spectra, green_spectra_err = continuum(spectra_green,
yerrs_green,
cont_mask=con_mask_green,
deg=deg)
red_spectra, red_spectra_err = continuum(spectra_red,
yerrs_red,
cont_mask=con_mask_red,
deg=deg)
normalized_spectra = np.concatenate(
(blue_spectra, green_spectra, red_spectra), axis=1)
normalized_spectra_err = np.concatenate(
(blue_spectra_err, green_spectra_err, red_spectra_err), axis=1)
# set negative flux and error as 0
normalized_spectra[normalized_spectra < 0.] = 0.
normalized_spectra_err[normalized_spectra < 0.] = 0.
# set inf and nan as 0
normalized_spectra[np.isinf(normalized_spectra)] = mask_value
normalized_spectra[np.isnan(normalized_spectra)] = mask_value
normalized_spectra_err[np.isinf(normalized_spectra)] = 0.
normalized_spectra_err[np.isnan(normalized_spectra)] = 0.
if bitmask is not None:
bitmask = gap_delete(bitmask, dr=dr)
if target_bit is None:
target_bit = [0, 1, 2, 3, 4, 5, 6, 7, 12]
mask = bitmask_boolean(bitmask, target_bit)
normalized_spectra[mask] = mask_value
normalized_spectra_err[mask] = mask_value
return normalized_spectra, normalized_spectra_err
def apogee_continuum(spectra,
spectra_err,
cont_mask=None,
deg=2,
dr=None,
bitmask=None,
target_bit=None,
mask_value=1):
"""
NAME:
apogee_continuum
PURPOSE:
apogee_continuum() is designed only for apogee spectra
Fit Chebyshev polynomials to the flux values in the continuum mask by chips. The resulting continuum will have
the same shape as `fluxes`.
INPUT:
spectra (ndaray): spectra
spectra_err (ndaray): spectra uncertainty (std/sigma)
cont_mask (ndaray): A mask for continuum pixels to use, or not specifying it to use mine
deg (int): The degree of Chebyshev polynomial to use in each region, default is 2 which works the best so far
dr (int): APOGEE DR, example dr=14
bitmask (ndarray or None): bitmask array of the spectra, same shape
target_bit (list): a list of bit to be masked
OUTPUT:
(ndarray): normalized flux
(ndarray): normalized error flux
HISTORY:
2018-Mar-21 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
spectra = gap_delete(spectra, dr=dr)
flux_errs = gap_delete(spectra_err, dr=dr)
spectra_blue, spectra_green, spectra_red = chips_split(spectra, dr=dr)
yerrs_blue, yerrs_green, yerrs_red = chips_split(flux_errs, dr=dr)
if cont_mask is None:
maskpath = os.path.join(os.path.dirname(astroNN.__path__[0]),
'astroNN', 'data', f'dr{dr}_contmask.npy')
cont_mask = np.load(maskpath)
con_mask_blue, con_mask_green, con_mask_red = chips_split(cont_mask, dr=dr)
con_mask_blue, con_mask_green, con_mask_red = con_mask_blue[
0], con_mask_green[0], con_mask_red[0]
# Continuum chips by chips
blue_spectra, blue_spectra_err = continuum(spectra_blue,
yerrs_blue,
cont_mask=con_mask_blue,
deg=deg)
green_spectra, green_spectra_err = continuum(spectra_green,
yerrs_green,
cont_mask=con_mask_green,
deg=deg)
red_spectra, red_spectra_err = continuum(spectra_red,
yerrs_red,
cont_mask=con_mask_red,
deg=deg)
normalized_spectra = np.concatenate(
(blue_spectra, green_spectra, red_spectra), axis=1)
normalized_spectra_err = np.concatenate(
(blue_spectra_err, green_spectra_err, red_spectra_err), axis=1)
if bitmask is not None:
bitmask = gap_delete(bitmask, dr=dr)
if target_bit is None:
target_bit = [0, 1, 2, 3, 4, 5, 6, 7, 12]
mask = np.invert(bitmask_boolean(bitmask, target_bit))
normalized_spectra[mask] = mask_value
normalized_spectra_err[mask] = mask_value
return normalized_spectra, normalized_spectra_err
def load_allstar(self):
self.apogee_dr = apogee_default_dr(dr=self.apogee_dr)
allstarpath = allstar(dr=self.apogee_dr)
hdulist = fits.open(allstarpath)
print(f'Loading allStar DR{self.apogee_dr} catalog')
return hdulist
def combined_spectra(
dr=None,
location=None,
field=None,
apogee=None,
telescope=None,
verbose=1,
flag=None,
):
"""
Download the required combined spectra file a.k.a aspcapStar
:param dr: APOGEE DR
:type dr: int
:param location: Location ID [Optional]
:type location: int
:param field: Field [Optional]
:type field: str
:param apogee: Apogee ID
:type apogee: str
:param telescope: Telescope ID, for example 'apo25m' or 'lco25m'
:type telescope: str
:param verbose: verbose, set 0 to silent most logging
:type verbose: int
:param flag: 0: normal, 1: force to re-download
:type flag: int
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History:
| 2017-Oct-15 - Written - Henry Leung (University of Toronto)
| 2018-Aug-31 - Updated - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
# for DR16=<, location is expected to be none because field is used
if (location is None and dr < 16) or (
field is None and dr >= 16): # try to load info if not enough info
global _ALLSTAR_TEMP
if not str(f"dr{dr}") in _ALLSTAR_TEMP:
_ALLSTAR_TEMP[f"dr{dr}"] = fits.getdata(allstar(dr=dr))
if telescope is None:
matched_idx = [
np.nonzero(_ALLSTAR_TEMP[f"dr{dr}"]["APOGEE_ID"] == apogee)[0]
][0]
else:
matched_idx = [
np.nonzero([
(_ALLSTAR_TEMP[f"dr{dr}"]["APOGEE_ID"] == apogee)
& (_ALLSTAR_TEMP[f"dr{dr}"]["TELESCOPE"] == telescope)
])
][0][1]
if len(matched_idx) == 0:
raise ValueError(
f"No entry found in allstar DR{dr} met with your requirement!!"
)
location = (_ALLSTAR_TEMP[f"dr{dr}"]["LOCATION_ID"][matched_idx][0]
if not location else location)
field = (_ALLSTAR_TEMP[f"dr{dr}"]["FIELD"][matched_idx][0]
if not field else field)
telescope = (_ALLSTAR_TEMP[f"dr{dr}"]["TELESCOPE"][matched_idx][0]
if not telescope else telescope)
if dr == 13:
reduce_prefix = "r6"
aspcap_code = "l30e"
str1 = f"https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/{location}/"
filename = f"aspcapStar-{reduce_prefix}-{aspcap_code}.2-{apogee}.fits"
hash_filename = f"stars_{aspcap_code}_{aspcap_code}.2_{location}.sha1sum"
urlstr = str1 + filename
# check folder existence
fullfoldername = os.path.join(
apogee_env(),
f"dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/",
str(location),
)
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
elif dr == 14:
reduce_prefix = "r8"
aspcap_code = "l31c"
str1 = f"https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/{location}/"
filename = f"aspcapStar-{reduce_prefix}-{aspcap_code}.2-{apogee}.fits"
hash_filename = f"stars_{aspcap_code}_{aspcap_code}.2_{location}.sha1sum"
urlstr = str1 + filename
# check folder existence
fullfoldername = os.path.join(
apogee_env(),
f"dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/",
str(location),
)
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
elif dr == 16:
reduce_prefix = "r12"
aspcap_code = "l33"
str1 = f"https://data.sdss.org/sas/dr16/apogee/spectro/aspcap/{reduce_prefix}/{aspcap_code}/{telescope}/{field}/"
filename = f"aspcapStar-{reduce_prefix}-{apogee}.fits"
hash_filename = f"{reduce_prefix}_{reduce_prefix}_{telescope}_{field}.sha1sum"
urlstr = str1 + filename
# check folder existence
fullfoldername = os.path.join(
apogee_env(),
f"dr{dr}/apogee/spectro/aspcap/{reduce_prefix}/{aspcap_code}/{telescope}",
str(f"{field}"),
)
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
else:
raise ValueError("combined_spectra() only supports APOGEE DR13-DR16")
# check hash file
full_hash_filename = os.path.join(fullfoldername, hash_filename)
if not os.path.isfile(full_hash_filename):
# return warning flag if the location_id cannot even be found
try:
urllib.request.urlopen(str1)
except urllib.error.HTTPError:
return warning_flag
urllib.request.urlretrieve(str1 + hash_filename, full_hash_filename)
hash_list = np.loadtxt(full_hash_filename, dtype="str").T
# In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too
file_hash = hash_list[0][np.argwhere(hash_list[1] == filename)]
if os.path.isfile(fullfilename) and flag is None:
checksum = filehash(fullfilename, algorithm="sha1")
if checksum != file_hash and len(file_hash) != 0:
warnings.warn(
"File corruption detected, astroNN is attempting to download again"
)
combined_spectra(dr=dr,
location=location,
apogee=apogee,
verbose=verbose,
flag=1)
if verbose == 1:
logging.info(fullfilename + " was found!")
elif not os.path.isfile(fullfilename) or flag == 1:
try:
urllib.request.urlretrieve(urlstr, fullfilename)
logging.info(
f"Downloaded DR{dr} combined file successfully to {fullfilename}"
)
checksum = filehash(fullfilename, algorithm="sha1")
if checksum != file_hash and len(file_hash) != 0:
warnings.warn(
"File corruption detected, astroNN is attempting to download again"
)
combined_spectra(dr=dr,
location=location,
apogee=apogee,
verbose=verbose,
flag=1)
except urllib.error.HTTPError as emsg:
if "401" in str(emsg):
fullfilename = __apogee_credentials_downloader(
urlstr, fullfilename)
elif "404" in str(emsg):
warnings.warn(f"{urlstr} cannot be found on server, skipped")
fullfilename = warning_flag
else:
warnings.warn(f"Unknown error occurred - {emsg}")
fullfilename = warning_flag
return fullfilename
def visit_spectra(dr=None, location=None, field=None, apogee=None, telescope=None, verbose=1, flag=None,
commission=False):
"""
Download the required individual spectra file a.k.a apStar or asStar
:param dr: APOGEE DR
:type dr: int
:param location: Location ID [Optional]
:type location: int
:param field: Field [Optional]
:type field: str
:param apogee: Apogee ID
:type apogee: str
:param telescope: Telescope ID, for example 'apo25m' or 'lco25m'
:type telescope: str
:param verbose: verbose, set 0 to silent most logging
:type verbose: int
:param flag: 0: normal, 1: force to re-download
:type flag: int
:param commission: whether the spectra is taken during commissioning
:type commission: bool
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History:
| 2017-Nov-11 - Written - Henry Leung (University of Toronto)
| 2018-Aug-31 - Updated - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
# for DR16=<, location is expected to be none because field is used
if (location is None and dr < 16) or (field is None and dr >= 16): # try to load info if not enough info
global _ALLSTAR_TEMP
if not str(f'dr{dr}') in _ALLSTAR_TEMP:
_ALLSTAR_TEMP[f'dr{dr}'] = fits.getdata(allstar(dr=dr))
if telescope is None:
matched_idx = [np.nonzero(_ALLSTAR_TEMP[f'dr{dr}']['APOGEE_ID'] == apogee)[0]][0]
else:
matched_idx = [np.nonzero([(_ALLSTAR_TEMP[f'dr{dr}']['APOGEE_ID'] == apogee) &
(_ALLSTAR_TEMP[f'dr{dr}']['TELESCOPE'] == telescope)])][0][1]
if len(matched_idx) == 0:
raise ValueError(f"No entry found in allstar DR{dr} met with your requirement!!")
location = _ALLSTAR_TEMP[f'dr{dr}']['LOCATION_ID'][matched_idx][0] if not location else location
field = _ALLSTAR_TEMP[f'dr{dr}']['FIELD'][matched_idx][0] if not field else field
telescope = _ALLSTAR_TEMP[f'dr{dr}']['TELESCOPE'][matched_idx][0] if not telescope else telescope
if dr == 13:
reduce_prefix = 'r6'
str1 = f'https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/{location}/'
if commission:
filename = f'apStarC-{reduce_prefix}-{apogee}.fits'
else:
filename = f'apStar-{reduce_prefix}-{apogee}.fits'
urlstr = str1 + filename
hash_filename = f'{reduce_prefix}_stars_apo25m_{location}.sha1sum'
fullfoldername = os.path.join(apogee_env(), f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/',
str(location))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
elif dr == 14:
reduce_prefix = 'r8'
str1 = f'https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/{location}/'
if commission:
filename = f'apStarC-{reduce_prefix}-{apogee}.fits'
else:
filename = f'apStar-{reduce_prefix}-{apogee}.fits'
urlstr = str1 + filename
hash_filename = f'{reduce_prefix}_stars_apo25m_{location}.sha1sum'
fullfoldername = os.path.join(apogee_env(), f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/',
str(location))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
elif dr == 16:
reduce_prefix = 'r12'
str1 = f'https://data.sdss.org/sas/dr16/apogee/spectro/redux/{reduce_prefix}/stars/{telescope}/{field}/'
if telescope == 'lco25m':
if commission:
filename = f'asStarC-{reduce_prefix}-{apogee}.fits'
else:
filename = f'asStar-{reduce_prefix}-{apogee}.fits'
else:
if commission:
filename = f'apStarC-{reduce_prefix}-{apogee}.fits'
else:
filename = f'apStar-{reduce_prefix}-{apogee}.fits'
urlstr = str1 + filename
hash_filename = f'{reduce_prefix}_stars_{telescope}_{field}.sha1sum'
fullfoldername = os.path.join(apogee_env(),
f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{telescope}/',
str(f'{field}'))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
else:
raise ValueError('visit_spectra() only supports DR13-DR16')
# check hash file
full_hash_filename = os.path.join(fullfoldername, hash_filename)
if not os.path.isfile(full_hash_filename):
# return warning flag if the location_id cannot even be found
try:
urllib.request.urlopen(str1)
except urllib.error.HTTPError:
return warning_flag
urllib.request.urlretrieve(str1 + hash_filename, full_hash_filename)
hash_list = np.loadtxt(full_hash_filename, dtype='str').T
fullfilename = os.path.join(fullfoldername, filename)
# In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too
# visit spectra has a different filename in checksum
# handle the case where apogee_id cannot be found
hash_idx = [i for i, item in enumerate(hash_list[1]) if f'apStar-{reduce_prefix}-{apogee}' in item]
file_hash = hash_list[0][hash_idx]
if os.path.isfile(fullfilename) and flag is None:
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash and len(file_hash) != 0:
print('File corruption detected, astroNN is attempting to download again')
visit_spectra(dr=dr, location=location, apogee=apogee, verbose=verbose, flag=1)
if verbose:
print(fullfilename + ' was found!')
elif not os.path.isfile(fullfilename) or flag == 1:
try:
urllib.request.urlretrieve(urlstr, fullfilename)
print(f'Downloaded DR{dr} individual visit file successfully to {fullfilename}')
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash and len(file_hash) != 0:
print('File corruption detected, astroNN is attempting to download again')
visit_spectra(dr=dr, location=location, apogee=apogee, verbose=verbose, flag=1)
except urllib.error.HTTPError as emsg:
if '401' in str(emsg):
fullfilename = __apogee_credentials_downloader(urlstr, fullfilename)
elif '404' in str(emsg):
print(f'{urlstr} cannot be found on server, skipped')
fullfilename = warning_flag
else:
print(f"Unknown error occurred - {emsg}")
fullfilename = warning_flag
return fullfilename
def visit_spectra(dr=None, location=None, apogee=None, verbose=1, flag=None):
"""
Download the required individual spectra file
:param dr: APOGEE DR
:type dr: int
:param location: Location ID [Optional]
:type location: int
:param apogee: Apogee ID
:type apogee: str
:param flag: 0: normal, 1: force to re-download
:type flag: int
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History: 2017-Nov-11 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if location is None:
global _ALLSTAR_TEMP
if _ALLSTAR_TEMP is None:
_ALLSTAR_TEMP = fits.getdata(allstar(dr=dr))
location = _ALLSTAR_TEMP['LOCATION_ID'][np.nonzero(
_ALLSTAR_TEMP['APOGEE_ID'] == apogee)][0]
if dr == 13:
reduce_prefix = 'r6'
str1 = f'https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/stars/apo25m/{location}/'
filename = f'apStar-r6-{apogee}.fits'
urlstr = str1 + filename
hash_filename = f'r6_stars_apo25m_{location}.sha1sum'
fullfoldername = os.path.join(
apogee_env(), 'dr13/apogee/spectro/redux/r6/stars/apo25m/',
str(location))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
# check hash file
full_hash_filename = os.path.join(fullfoldername, hash_filename)
if not os.path.isfile(full_hash_filename):
# return warning flag if the location_id cannot even be found
try:
urllib.request.urlopen(str1)
except urllib.request.HTTPError:
return warning_flag
urllib.request.urlretrieve(str1 + hash_filename,
full_hash_filename)
hash_list = np.loadtxt(full_hash_filename, dtype='str').T
fullfilename = os.path.join(
apogee_env(), 'dr13/apogee/spectro/redux/r6/stars/apo25m/',
str(location), filename)
elif dr == 14:
reduce_prefix = 'r8'
str1 = f'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/stars/apo25m/{location}/'
filename = f'apStar-r8-{apogee}.fits'
urlstr = str1 + filename
hash_filename = f'r8_stars_apo25m_{location}.sha1sum'
fullfoldername = os.path.join(
apogee_env(), 'dr14/apogee/spectro/redux/r8/stars/apo25m/',
str(location))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
# check hash file
full_hash_filename = os.path.join(fullfoldername, hash_filename)
if not os.path.isfile(full_hash_filename):
# return warning flag if the location_id cannot even be found
try:
urllib.request.urlopen(str1)
except urllib.request.HTTPError:
return warning_flag
urllib.request.urlretrieve(str1 + hash_filename,
full_hash_filename)
hash_list = np.loadtxt(full_hash_filename, dtype='str').T
fullfilename = os.path.join(
apogee_env(), 'dr14/apogee/spectro/redux/r8/stars/apo25m/',
str(location), filename)
else:
raise ValueError('visit_spectra() only supports DR13 or DR14')
# In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too
# visit spectra has a different filename in checksum
# handle the case where apogee_id cannot be found
hash_idx = [
i for i, item in enumerate(hash_list[1])
if f'apStar-{reduce_prefix}-{apogee}' in item
]
file_hash = hash_list[0][hash_idx]
if os.path.isfile(fullfilename) and flag is None:
checksum = sha1_checksum(fullfilename)
if checksum != file_hash and len(file_hash) != 0:
print(
'File corruption detected, astroNN attempting to download again'
)
visit_spectra(dr=dr,
location=location,
apogee=apogee,
verbose=verbose,
flag=1)
if verbose == 1:
print(fullfilename + ' was found!')
elif not os.path.isfile(fullfilename) or flag == 1:
try:
urllib.request.urlretrieve(urlstr, fullfilename)
print(
f'Downloaded DR14 individual visit file successfully to {fullfilename}'
)
checksum = sha1_checksum(fullfilename)
if checksum != file_hash and len(file_hash) != 0:
print(
'File corruption detected, astroNN attempting to download again'
)
visit_spectra(dr=dr,
location=location,
apogee=apogee,
verbose=verbose,
flag=1)
except urllib.request.HTTPError:
print(f'{urlstr} cannot be found on server, skipped')
fullfilename = warning_flag
return fullfilename
def allstar(dr=None, flag=None):
"""
Download the allStar file (catalog of ASPCAP stellar parameters and abundances from combined spectra)
:param dr: APOGEE DR
:type dr: int
:param flag: 0: normal, 1: force to re-download
:type flag: int
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History: 2017-Oct-09 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 13:
file_hash = '1718723ada3018de94e1022cd57d4d950a74f91f'
# Check if directory exists
fullfoldername = os.path.join(apogee_env(), 'dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/')
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
filename = 'allStar-l30e.2.fits'
fullfilename = os.path.join(fullfoldername, filename)
url = f'https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/{filename}'
elif dr == 14:
file_hash = 'a7e1801924661954da792e377ad54f412219b105'
fullfoldername = os.path.join(apogee_env(), 'dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/')
# Check if directory exists
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
filename = 'allStar-l31c.2.fits'
fullfilename = os.path.join(fullfoldername, filename)
url = f'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/{filename}'
elif dr == 16:
file_hash = '66fe854bd000ca1c0a6b50a998877e4a3e41d184'
fullfoldername = os.path.join(apogee_env(), 'dr16/apogee/spectro/aspcap/r12/l33/')
# Check if directory exists
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
filename = 'allStar-r12-l33.fits'
fullfilename = os.path.join(fullfoldername, filename)
url = f'https://data.sdss.org/sas/dr16/apogee/spectro/aspcap/r12/l33/{filename}'
else:
raise ValueError('allstar() only supports APOGEE DR13-DR16')
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash.lower():
print('File corruption detected, astroNN is attempting to download again')
allstar(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
# Check if files exists
if not os.path.isfile(os.path.join(fullfoldername, filename)) or flag == 1:
with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:
try:
urllib.request.urlretrieve(url, fullfilename, reporthook=t.update_to)
print(f'Downloaded DR{dr:d} allStar file catalog successfully to {fullfilename}')
checksum = filehash(fullfilename, algorithm='sha1')
if checksum != file_hash.lower():
print('File corruption detected, astroNN is attempting to download again')
allstar(dr=dr, flag=1)
except urllib.error.HTTPError as emsg:
if '401' in str(emsg):
fullfilename = __apogee_credentials_downloader(url, fullfilename)
elif '404' in str(emsg):
print(f'{url} cannot be found on server, skipped')
fullfilename = warning_flag
else:
print(f"Unknown error occurred - {emsg}")
fullfilename = warning_flag
return fullfilename
def apogee_astronn(dr=None, flag=None):
"""
Download the apogee_astroNN file (catalog of astroNN stellar parameters, abundances, distances and orbital
parameters from combined spectra)
:param dr: APOGEE DR
:type dr: int
:param flag: 0: normal, 1: force to re-download
:type flag: int
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History: 2019-Dec-10 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 16:
# Check if directory exists
fullfoldername = os.path.join(apogee_env(),
"dr16/apogee/vac/apogee-astronn/")
# Check if directory exists
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
filename = "apogee_astroNN-DR16-v1.fits"
fullfilename = os.path.join(fullfoldername, filename)
file_hash = "1b81ed13eef36fe9a327a05f4a622246522199b2"
url = f"https://data.sdss.org/sas/dr16/apogee/vac/apogee-astronn/{filename}"
elif dr == 17:
# Check if directory exists
fullfoldername = os.path.join(apogee_env(),
"dr17/apogee/vac/apogee-astronn/")
# Check if directory exists
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
filename = "apogee_astroNN-DR17.fits"
fullfilename = os.path.join(fullfoldername, filename)
file_hash = "c422b9adba840b3415af2fe6dec6500219f1b68f"
url = f"https://data.sdss.org/sas/dr17/apogee/vac/apogee-astronn/{filename}"
else:
raise ValueError("apogee_astroNN() only supports APOGEE DR16-DR17")
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = filehash(fullfilename, algorithm="sha1")
if checksum != file_hash.lower():
warnings.warn(
"File corruption detected, astroNN is attempting to download again"
)
apogee_astronn(dr=dr, flag=1)
else:
logging.info(fullfilename + " was found!")
# Check if files exists
if not os.path.isfile(os.path.join(fullfoldername, filename)) or flag == 1:
with TqdmUpTo(unit="B",
unit_scale=True,
miniters=1,
desc=url.split("/")[-1]) as t:
try:
urllib.request.urlretrieve(url,
fullfilename,
reporthook=t.update_to)
logging.info(
f"Downloaded DR{dr:d} apogee_astroNN file catalog successfully to {fullfilename}"
)
checksum = filehash(fullfilename, algorithm="sha1")
if checksum != file_hash.lower():
warnings.warn(
"File corruption detected, astroNN is attempting to download again"
)
apogee_astronn(dr=dr, flag=1)
except urllib.error.HTTPError as emsg:
if "401" in str(emsg):
fullfilename = __apogee_credentials_downloader(
url, fullfilename)
elif "404" in str(emsg):
warnings.warn(f"{url} cannot be found on server, skipped")
fullfilename = warning_flag
else:
warnings.warn(f"Unknown error occurred - {emsg}")
fullfilename = warning_flag
return fullfilename
def allstar(dr=None, flag=None):
"""
NAME:
allstar
PURPOSE:
download the allStar file (catalog of ASPCAP stellar parameters and abundances from combined spectra)
INPUT:
dr (int): APOGEE DR, example dr=14
flag (int): 0: normal, 1: force to re-download
OUTPUT:
(path): full file path and download in background
HISTORY:
2017-Oct-09 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 13:
file_hash = '1718723ada3018de94e1022cd57d4d950a74f91f'
# Check if directory exists
fullfoldername = os.path.join(
apogee_env(), 'dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/')
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
filename = 'allStar-l30e.2.fits'
fullfilename = os.path.join(fullfoldername, filename)
url = f'https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/{filename}'
elif dr == 14:
file_hash = 'a7e1801924661954da792e377ad54f412219b105'
fullfoldername = os.path.join(
apogee_env(), 'dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/')
# Check if directory exists
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
filename = 'allStar-l31c.2.fits'
fullfilename = os.path.join(fullfoldername, filename)
url = f'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/{filename}'
else:
raise ValueError('allstar() only supports APOGEE DR13 and DR14')
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print(
'File corruption detected, astroNN attempting to download again'
)
allstar(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
# Check if files exists
if not os.path.isfile(os.path.join(fullfoldername, filename)) or flag == 1:
with TqdmUpTo(unit='B',
unit_scale=True,
miniters=1,
desc=url.split('/')[-1]) as t:
urllib.request.urlretrieve(url,
fullfilename,
reporthook=t.update_to)
print(
f'Downloaded DR{dr:d} allStar file catalog successfully to {fullfilename}'
)
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print(
'File corruption detected, astroNN attempting to download again'
)
allstar(dr=dr, flag=1)
return fullfilename
def apogee_rc(dr=None, flag=None):
"""
Download the APOGEE red clumps catalogue
:param dr: Apogee DR
:type dr: int
:param flag: Force to download if flag=1
:type flag: int
:return: full file path
:rtype: str
:History: 2017-Nov-16 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 13:
file_hash = "5e87eb3ba202f9db24216978dafb19d39d382fc6"
str1 = "https://data.sdss.org/sas/dr13/apogee/vac/apogee-rc/cat/"
filename = f"apogee-rc-DR{dr}.fits"
urlstr = str1 + filename
fullfoldername = os.path.join(apogee_env(),
"dr13/apogee/vac/apogee-rc/cat/")
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
elif dr == 14:
file_hash = "104513070f1c280954f3d1886cac429dbdf2eaf6"
str1 = "https://data.sdss.org/sas/dr14/apogee/vac/apogee-rc/cat/"
filename = f"apogee-rc-DR{dr}.fits"
urlstr = str1 + filename
fullfoldername = os.path.join(apogee_env(),
"dr14/apogee/vac/apogee-rc/cat/")
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
elif dr == 16:
file_hash = "0bc75a230058f50ed8a5ea3fa8554d803ffc103d"
str1 = "https://data.sdss.org/sas/dr16/apogee/vac/apogee-rc/cat/"
filename = f"apogee-rc-DR{dr}.fits"
urlstr = str1 + filename
fullfoldername = os.path.join(apogee_env(),
"dr16/apogee/vac/apogee-rc/cat/")
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
elif dr == 17:
file_hash = "d54e0ea4e6a3f5cc3c02a73b93260e992d9836d0"
str1 = "https://data.sdss.org/sas/dr17/apogee/vac/apogee-rc/cat/"
filename = f"apogee-rc-DR{dr}.fits"
urlstr = str1 + filename
fullfoldername = os.path.join(apogee_env(),
"dr17/apogee/vac/apogee-rc/cat/")
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(fullfoldername, filename)
else:
raise ValueError("apogee_rc() only supports APOGEE DR13-DR17")
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = filehash(fullfilename, algorithm="sha1")
if checksum != file_hash.lower():
warnings.warn(
"File corruption detected, astroNN is attempting to download again"
)
apogee_rc(dr=dr, flag=1)
else:
logging.info(fullfilename + " was found!")
elif not os.path.isfile(fullfilename) or flag == 1:
try:
with TqdmUpTo(unit="B",
unit_scale=True,
miniters=1,
desc=urlstr.split("/")[-1]) as t:
urllib.request.urlretrieve(urlstr,
fullfilename,
reporthook=t.update_to)
logging.info(
f"Downloaded DR{dr} Red Clumps Catalog successfully to {fullfilename}"
)
checksum = filehash(fullfilename, algorithm="sha1")
if checksum != file_hash.lower():
warnings.warn(
"File corruption detected, astroNN is attempting to download again"
)
apogee_rc(dr=dr, flag=1)
except urllib.error.HTTPError as emsg:
if "401" in str(emsg):
fullfilename = __apogee_credentials_downloader(
urlstr, fullfilename)
elif "404" in str(emsg):
warnings.warn(f"{urlstr} cannot be found on server, skipped")
fullfilename = warning_flag
else:
warnings.warn(f"Unknown error occurred - {emsg}")
fullfilename = warning_flag
return fullfilename
def apogee_vac_rc(dr=None, flag=None):
"""
NAME:
apogee_vac_rc
PURPOSE:
download the red clumps catalogue
INPUT:
dr (int): APOGEE DR, example dr=14
flag (int): 0: normal, 1: force to re-download
OUTPUT:
(path): full file path and download in background
HISTORY:
2017-Nov-16 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 13:
file_hash = '5e87eb3ba202f9db24216978dafb19d39d382fc6'
str1 = 'https://data.sdss.org/sas/dr13/apogee/vac/apogee-rc/cat/'
filename = f'apogee-rc-DR{dr}.fits'
urlstr = str1 + filename
fullfilename = os.path.join(apogee_env(),
'dr13/apogee/vac/apogee-rc/cat/')
if not os.path.exists(fullfilename):
os.makedirs(fullfilename)
fullfilename = os.path.join(apogee_env(),
'dr13/apogee/vac/apogee-rc/cat/', filename)
elif dr == 14:
file_hash = '104513070f1c280954f3d1886cac429dbdf2eaf6'
str1 = 'https://data.sdss.org/sas/dr14/apogee/vac/apogee-rc/cat/'
filename = f'apogee-rc-DR{dr}.fits'
urlstr = str1 + filename
fullfilename = os.path.join(apogee_env(),
'dr14/apogee/vac/apogee-rc/cat/')
if not os.path.exists(fullfilename):
os.makedirs(fullfilename)
fullfilename = os.path.join(apogee_env(),
'dr14/apogee/vac/apogee-rc/cat/', filename)
else:
raise ValueError('apogee_vac_rc() only supports DR13 or DR14')
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print(
'File corruption detected, astroNN attempting to download again'
)
apogee_vac_rc(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
elif not os.path.isfile(fullfilename) or flag == 1:
try:
with TqdmUpTo(unit='B',
unit_scale=True,
miniters=1,
desc=urlstr.split('/')[-1]) as t:
urllib.request.urlretrieve(urlstr,
fullfilename,
reporthook=t.update_to)
print(
f'Downloaded DR{dr} Red Clumps Catalog successfully to {fullfilename}'
)
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print(
'File corruption detected, astroNN attempting to download again'
)
apogee_vac_rc(dr=dr, flag=1)
except urllib.request.HTTPError:
print(f'{urlstr} cannot be found on server, skipped')
fullfilename = warning_flag
return fullfilename
def apokasc_logg(dr=None, h5name=None, folder_name=None):
"""
NAME: apokasc_logg
PURPOSE: check apokasc result
INPUT:
dr = 14
folder_name = the folder name contains the model
h5name = name of h5 dataset you want to create
OUTPUT: {h5name}_train.h5 {h5name}_test.h5
HISTORY:
2017-Nov-15 Henry Leung
"""
# prevent Tensorflow taking up all the GPU memory
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
set_session(tf.Session(config=config))
h5name_check(h5name)
dr = apogee_default_dr(dr=dr)
catalog_list = Vizier.find_catalogs('apokasc')
Vizier.ROW_LIMIT = 5000
catalogs = Vizier.get_catalogs(catalog_list.keys())[1]
apokasc_ra = catalogs['_RA']
apokasc_dec = catalogs['_DE']
apokasc_logg = catalogs['log_g_']
allstarpath = allstar(dr=dr)
hdulist = fits.open(allstarpath)
print('Now processing allStar DR{} catalog'.format(dr))
cannon_fullfilename = allstarcannon(dr=14)
cannonhdulist = fits.open(cannon_fullfilename)
apogee_ra = hdulist[1].data['RA']
apogee_dec = hdulist[1].data['DEC']
m1, m2, sep = xmatch(apogee_ra, apokasc_ra, maxdist=2, colRA1=apogee_ra, colDec1=apogee_dec, epoch1=2000.,
colRA2=apokasc_ra, colDec2=apokasc_dec, epoch2=2000., colpmRA2=None, colpmDec2=None, swap=True)
currentdir = os.getcwd()
fullfolderpath = currentdir + '/' + folder_name
modelname = '/model_{}.h5'.format(folder_name[-11:])
model = load_model(os.path.normpath(fullfolderpath + modelname))
mean_and_std = np.load(fullfolderpath + '/meanstd.npy')
spec_meanstd = np.load(fullfolderpath + '/spectra_meanstd.npy')
apokasc_logg = apokasc_logg[m2]
aspcap_residue = []
astronn_residue = []
cannon_residue = []
for counter, index in enumerate(m1):
apogee_id = hdulist[1].data['APOGEE_ID'][index]
location_id = hdulist[1].data['LOCATION_ID'][index]
cannon_residue.extend([cannonhdulist[1].data['LOGG'][index] - apokasc_logg[counter]])
warningflag, path = combined_spectra(dr=dr, location=location_id, apogee=apogee_id, verbose=0)
if warningflag is None:
combined_file = fits.open(path)
_spec = combined_file[1].data # Pseudo-comtinumm normalized flux
spec = (_spec - spec_meanstd[0])/spec_meanstd[1]
spec = gap_delete(spec, dr=14)
aspcap_residue.extend([hdulist[1].data['PARAM'][index, 1] - apokasc_logg[counter]])
prediction = model.predict(spec.reshape([1, len(spec), 1]), batch_size=1)
prediction *= mean_and_std[1]
prediction += mean_and_std[0]
astronn_residue.extend([prediction[0,1] - apokasc_logg[counter]])
hdulist.close()
cannonhdulist.close()
plt.figure(figsize=(15, 11), dpi=200)
plt.axhline(0, ls='--', c='k', lw=2)
plt.scatter(apokasc_logg, aspcap_residue, s=3)
fullname = 'Log(g)'
x_lab = 'APOKASC'
y_lab = 'ASPCAP'
plt.xlabel('APOKASC ' + fullname, fontsize=25)
plt.ylabel('$\Delta$ ' + fullname + '\n(' + y_lab + ' - ' + x_lab + ')', fontsize=25)
plt.tick_params(labelsize=20, width=1, length=10)
plt.xlim([np.min(apokasc_logg), np.max(apokasc_logg)])
ranges = (np.max(apokasc_logg) - np.min(apokasc_logg)) / 2
plt.ylim([-ranges, ranges])
bbox_props = dict(boxstyle="square,pad=0.3", fc="w", ec="k", lw=2)
plt.figtext(0.6, 0.75,
'$\widetilde{m}$=' + '{0:.3f}'.format(np.median(aspcap_residue)) + ' s=' +
'{0:.3f}'.format(mad_std(aspcap_residue)), size=25, bbox=bbox_props)
plt.tight_layout()
plt.savefig(fullfolderpath + '/apokasc_aspcap_logg.png')
plt.close('all')
plt.clf()
plt.figure(figsize=(15, 11), dpi=200)
plt.axhline(0, ls='--', c='k', lw=2)
plt.scatter(apokasc_logg, astronn_residue, s=3)
fullname = 'Log(g)'
x_lab = 'APOKASC'
y_lab = 'astroNN'
plt.xlabel('APOKASC ' + fullname, fontsize=25)
plt.ylabel('$\Delta$ ' + fullname + '\n(' + y_lab + ' - ' + x_lab + ')', fontsize=25)
plt.tick_params(labelsize=20, width=1, length=10)
plt.xlim([np.min(apokasc_logg), np.max(apokasc_logg)])
ranges = (np.max(apokasc_logg) - np.min(apokasc_logg)) / 2
plt.ylim([-ranges, ranges])
bbox_props = dict(boxstyle="square,pad=0.3", fc="w", ec="k", lw=2)
plt.figtext(0.6, 0.75,
'$\widetilde{m}$=' + '{0:.3f}'.format(np.median(astronn_residue)) + ' s=' +
'{0:.3f}'.format(mad_std(astronn_residue)), size=25, bbox=bbox_props)
plt.tight_layout()
plt.savefig(fullfolderpath + '/apokasc_astroNN_logg.png')
plt.close('all')
plt.clf()
plt.figure(figsize=(15, 11), dpi=200)
plt.axhline(0, ls='--', c='k', lw=2)
plt.scatter(apokasc_logg, cannon_residue, s=3)
fullname = 'Log(g)'
x_lab = 'APOKASC'
y_lab = 'Cannon'
plt.xlabel('APOKASC ' + fullname, fontsize=25)
plt.ylabel('$\Delta$ ' + fullname + '\n(' + y_lab + ' - ' + x_lab + ')', fontsize=25)
plt.tick_params(labelsize=20, width=1, length=10)
plt.xlim([np.min(apokasc_logg), np.max(apokasc_logg)])
ranges = (np.max(apokasc_logg) - np.min(apokasc_logg)) / 2
plt.ylim([-ranges, ranges])
bbox_props = dict(boxstyle="square,pad=0.3", fc="w", ec="k", lw=2)
plt.figtext(0.6, 0.75,
'$\widetilde{m}$=' + '{0:.3f}'.format(np.median(cannon_residue)) + ' s=' +
'{0:.3f}'.format(mad_std(cannon_residue)), size=25, bbox=bbox_props)
plt.tight_layout()
plt.savefig(fullfolderpath + '/apokasc_cannon_logg.png')
plt.close('all')
plt.clf()
