def allvisit(dr=None, flag=None):
"""
Download the allVisit file (catalog of properties from individual visit spectra)
:param dr: APOGEE DR
:type dr: int
:param flag: 0: normal, 1: force to re-download
:type flag: int
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History: 2017-Oct-11 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 13:
file_hash = '2a3b13ccd40a2c8aea8321be9630117922d55b51'
# Check if directory exists
fullfilepath = os.path.join(apogee_env(), 'dr13/apogee/spectro/redux/r6/')
if not os.path.exists(fullfilepath):
os.makedirs(fullfilepath)
filename = 'allVisit-l30e.2.fits'
fullfilename = os.path.join(fullfilepath, filename)
url = f'https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/{filename}'
elif dr == 14:
file_hash = 'abcecbcdc5fe8d00779738702c115633811e6bbd'
# Check if directory exists
fullfilepath = os.path.join(apogee_env(), 'dr14/apogee/spectro/redux/r8/')
if not os.path.exists(fullfilepath):
os.makedirs(fullfilepath)
filename = 'allVisit-l31c.2.fits'
fullfilename = os.path.join(fullfilepath, filename)
url = f'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/{filename}'
else:
raise ValueError('allvisit() only supports APOGEE DR13-DR15')
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print('File corruption detected, astroNN attempting to download again')
allvisit(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
elif not os.path.isfile(os.path.join(fullfilepath, filename)) or flag == 1:
with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:
urllib.request.urlretrieve(url, fullfilename, reporthook=t.update_to)
print(f'Downloaded DR{dr:d} allVisit file catalog successfully to {fullfilepath}')
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print('File corruption detected, astroNN attempting to download again')
allstar(dr=dr, flag=1)
return fullfilename
def apogee_distances(dr=None, flag=None):
"""
Download the Apogee Distances catalogue
:param dr: Apogee DR
:type dr: int
:param flag: Force to download if flag=1
:type flag: int
:return: full file path
:rtype: str
:History: 2018-Jan-24 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 14:
file_hash = 'b33c8419be784b1be3d14af3ee9696c6ac31830f'
str1 = 'https://data.sdss.org/sas/dr14/apogee/vac/apogee-distances/'
filename = f'apogee_distances-DR{dr}.fits'
urlstr = str1 + filename
fullfilename = os.path.join(apogee_env(), 'dr14/apogee/vac/apogee-distances/')
if not os.path.exists(fullfilename):
os.makedirs(fullfilename)
fullfilename = os.path.join(apogee_env(), 'dr14/apogee/vac/apogee-distances/', filename)
else:
raise ValueError('apogee_distances() only supports DR14')
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print('File corruption detected, astroNN attempting to download again')
apogee_distances(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
elif not os.path.isfile(fullfilename) or flag == 1:
try:
with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=urlstr.split('/')[-1]) as t:
urllib.request.urlretrieve(urlstr, fullfilename, reporthook=t.update_to)
print(f'Downloaded DR{dr} Distances successfully to {fullfilename}')
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print('File corruption detected, astroNN attempting to download again')
apogee_distances(dr=dr, flag=1)
except urllib.request.HTTPError:
print(f'{urlstr} cannot be found on server, skipped')
fullfilename = warning_flag
return fullfilename
def allstarcannon(dr=None, flag=None):
"""
Download the allStarCannon file (catalog of Cannon stellar parameters and abundances from combined spectra)
:param dr: APOGEE DR
:type dr: int
:param flag: 0: normal, 1: force to re-download
:type flag: int
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History: 2017-Oct-24 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 14:
# Check if directory exists
fullfoldername = os.path.join(apogee_env(), 'dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/cannon/')
# Check if directory exists
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
filename = 'allStarCannon-l31c.2.fits'
fullfilename = os.path.join(fullfoldername, filename)
file_hash = '64d485e95b3504df0b795ab604e21a71d5c7ae45'
url = f'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/cannon/{filename}'
else:
raise ValueError('allstarcannon() only supports APOGEE DR14-DR15')
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print('File corruption detected, astroNN attempting to download again')
allstarcannon(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
# Check if files exists
if not os.path.isfile(os.path.join(fullfoldername, filename)) or flag == 1:
with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:
urllib.request.urlretrieve(url, fullfilename, reporthook=t.update_to)
print(f'Downloaded DR{dr:d} allStarCannon file catalog successfully to {fullfilename}')
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print('File corruption detected, astroNN attempting to download again')
allstarcannon(dr=dr, flag=1)
return fullfilename
def test_checksum(self):
import astroNN
from astroNN.shared.downloader_tools import md5_checksum, sha1_checksum, sha256_checksum
anderson2017_path = os.path.join(os.path.dirname(astroNN.__path__[0]), 'astroNN', 'data',
'anderson_2017_dr14_parallax.npz')
md5_pred = md5_checksum(anderson2017_path)
sha1_pred = sha1_checksum(anderson2017_path)
sha256_pred = sha256_checksum(anderson2017_path)
# read answer hashed by Windows Get-FileHash
self.assertEqual(md5_pred, '9C714F5FE22BB7C4FF9EA32F3E859D73'.lower())
self.assertEqual(sha1_pred, '733C0227CF93DB0CD6106B5349402F251E7ED735'.lower())
self.assertEqual(sha256_pred, '36C265C907F440114D747DA21D2A014D32B5E442D541F183C0EE862F5865FD26'.lower())
def apogee_vac_rc(dr=None, flag=None):
"""
NAME:
apogee_vac_rc
PURPOSE:
download the red clumps catalogue
INPUT:
dr (int): APOGEE DR, example dr=14
flag (int): 0: normal, 1: force to re-download
OUTPUT:
(path): full file path and download in background
HISTORY:
2017-Nov-16 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 13:
file_hash = '5e87eb3ba202f9db24216978dafb19d39d382fc6'
str1 = 'https://data.sdss.org/sas/dr13/apogee/vac/apogee-rc/cat/'
filename = f'apogee-rc-DR{dr}.fits'
urlstr = str1 + filename
fullfilename = os.path.join(apogee_env(),
'dr13/apogee/vac/apogee-rc/cat/')
if not os.path.exists(fullfilename):
os.makedirs(fullfilename)
fullfilename = os.path.join(apogee_env(),
'dr13/apogee/vac/apogee-rc/cat/', filename)
elif dr == 14:
file_hash = '104513070f1c280954f3d1886cac429dbdf2eaf6'
str1 = 'https://data.sdss.org/sas/dr14/apogee/vac/apogee-rc/cat/'
filename = f'apogee-rc-DR{dr}.fits'
urlstr = str1 + filename
fullfilename = os.path.join(apogee_env(),
'dr14/apogee/vac/apogee-rc/cat/')
if not os.path.exists(fullfilename):
os.makedirs(fullfilename)
fullfilename = os.path.join(apogee_env(),
'dr14/apogee/vac/apogee-rc/cat/', filename)
else:
raise ValueError('apogee_vac_rc() only supports DR13 or DR14')
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print(
'File corruption detected, astroNN attempting to download again'
)
apogee_vac_rc(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
elif not os.path.isfile(fullfilename) or flag == 1:
try:
with TqdmUpTo(unit='B',
unit_scale=True,
miniters=1,
desc=urlstr.split('/')[-1]) as t:
urllib.request.urlretrieve(urlstr,
fullfilename,
reporthook=t.update_to)
print(
f'Downloaded DR{dr} Red Clumps Catalog successfully to {fullfilename}'
)
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print(
'File corruption detected, astroNN attempting to download again'
)
apogee_vac_rc(dr=dr, flag=1)
except urllib.request.HTTPError:
print(f'{urlstr} cannot be found on server, skipped')
fullfilename = warning_flag
return fullfilename
def visit_spectra(dr=None, location=None, apogee=None, verbose=1, flag=None):
"""
Download the required individual spectra file
:param dr: APOGEE DR
:type dr: int
:param location: Location ID [Optional]
:type location: int
:param apogee: Apogee ID
:type apogee: str
:param flag: 0: normal, 1: force to re-download
:type flag: int
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History: 2017-Nov-11 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if location is None:
global _ALLSTAR_TEMP
if _ALLSTAR_TEMP is None:
_ALLSTAR_TEMP = fits.getdata(allstar(dr=dr))
location = _ALLSTAR_TEMP['LOCATION_ID'][np.nonzero(
_ALLSTAR_TEMP['APOGEE_ID'] == apogee)][0]
if dr == 13:
reduce_prefix = 'r6'
str1 = f'https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/stars/apo25m/{location}/'
filename = f'apStar-r6-{apogee}.fits'
urlstr = str1 + filename
hash_filename = f'r6_stars_apo25m_{location}.sha1sum'
fullfoldername = os.path.join(
apogee_env(), 'dr13/apogee/spectro/redux/r6/stars/apo25m/',
str(location))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
# check hash file
full_hash_filename = os.path.join(fullfoldername, hash_filename)
if not os.path.isfile(full_hash_filename):
# return warning flag if the location_id cannot even be found
try:
urllib.request.urlopen(str1)
except urllib.request.HTTPError:
return warning_flag
urllib.request.urlretrieve(str1 + hash_filename,
full_hash_filename)
hash_list = np.loadtxt(full_hash_filename, dtype='str').T
fullfilename = os.path.join(
apogee_env(), 'dr13/apogee/spectro/redux/r6/stars/apo25m/',
str(location), filename)
elif dr == 14:
reduce_prefix = 'r8'
str1 = f'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/stars/apo25m/{location}/'
filename = f'apStar-r8-{apogee}.fits'
urlstr = str1 + filename
hash_filename = f'r8_stars_apo25m_{location}.sha1sum'
fullfoldername = os.path.join(
apogee_env(), 'dr14/apogee/spectro/redux/r8/stars/apo25m/',
str(location))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
# check hash file
full_hash_filename = os.path.join(fullfoldername, hash_filename)
if not os.path.isfile(full_hash_filename):
# return warning flag if the location_id cannot even be found
try:
urllib.request.urlopen(str1)
except urllib.request.HTTPError:
return warning_flag
urllib.request.urlretrieve(str1 + hash_filename,
full_hash_filename)
hash_list = np.loadtxt(full_hash_filename, dtype='str').T
fullfilename = os.path.join(
apogee_env(), 'dr14/apogee/spectro/redux/r8/stars/apo25m/',
str(location), filename)
else:
raise ValueError('visit_spectra() only supports DR13 or DR14')
# In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too
# visit spectra has a different filename in checksum
# handle the case where apogee_id cannot be found
hash_idx = [
i for i, item in enumerate(hash_list[1])
if f'apStar-{reduce_prefix}-{apogee}' in item
]
file_hash = hash_list[0][hash_idx]
if os.path.isfile(fullfilename) and flag is None:
checksum = sha1_checksum(fullfilename)
if checksum != file_hash and len(file_hash) != 0:
print(
'File corruption detected, astroNN attempting to download again'
)
visit_spectra(dr=dr,
location=location,
apogee=apogee,
verbose=verbose,
flag=1)
if verbose == 1:
print(fullfilename + ' was found!')
elif not os.path.isfile(fullfilename) or flag == 1:
try:
urllib.request.urlretrieve(urlstr, fullfilename)
print(
f'Downloaded DR14 individual visit file successfully to {fullfilename}'
)
checksum = sha1_checksum(fullfilename)
if checksum != file_hash and len(file_hash) != 0:
print(
'File corruption detected, astroNN attempting to download again'
)
visit_spectra(dr=dr,
location=location,
apogee=apogee,
verbose=verbose,
flag=1)
except urllib.request.HTTPError:
print(f'{urlstr} cannot be found on server, skipped')
fullfilename = warning_flag
return fullfilename
def allstar(dr=None, flag=None):
"""
NAME:
allstar
PURPOSE:
download the allStar file (catalog of ASPCAP stellar parameters and abundances from combined spectra)
INPUT:
dr (int): APOGEE DR, example dr=14
flag (int): 0: normal, 1: force to re-download
OUTPUT:
(path): full file path and download in background
HISTORY:
2017-Oct-09 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 13:
file_hash = '1718723ada3018de94e1022cd57d4d950a74f91f'
# Check if directory exists
fullfoldername = os.path.join(
apogee_env(), 'dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/')
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
filename = 'allStar-l30e.2.fits'
fullfilename = os.path.join(fullfoldername, filename)
url = f'https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/{filename}'
elif dr == 14:
file_hash = 'a7e1801924661954da792e377ad54f412219b105'
fullfoldername = os.path.join(
apogee_env(), 'dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/')
# Check if directory exists
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
filename = 'allStar-l31c.2.fits'
fullfilename = os.path.join(fullfoldername, filename)
url = f'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/{filename}'
else:
raise ValueError('allstar() only supports APOGEE DR13 and DR14')
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print(
'File corruption detected, astroNN attempting to download again'
)
allstar(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
# Check if files exists
if not os.path.isfile(os.path.join(fullfoldername, filename)) or flag == 1:
with TqdmUpTo(unit='B',
unit_scale=True,
miniters=1,
desc=url.split('/')[-1]) as t:
urllib.request.urlretrieve(url,
fullfilename,
reporthook=t.update_to)
print(
f'Downloaded DR{dr:d} allStar file catalog successfully to {fullfilename}'
)
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print(
'File corruption detected, astroNN attempting to download again'
)
allstar(dr=dr, flag=1)
return fullfilename
def allstar(dr=None, flag=None):
"""
Download the allStar file (catalog of ASPCAP stellar parameters and abundances from combined spectra)
:param dr: APOGEE DR
:type dr: int
:param flag: 0: normal, 1: force to re-download
:type flag: int
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History: 2017-Oct-09 - Written - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if dr == 13:
file_hash = '1718723ada3018de94e1022cd57d4d950a74f91f'
# Check if directory exists
fullfoldername = os.path.join(apogee_env(), 'dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/')
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
filename = 'allStar-l30e.2.fits'
fullfilename = os.path.join(fullfoldername, filename)
url = f'https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/{filename}'
elif dr == 14:
file_hash = 'a7e1801924661954da792e377ad54f412219b105'
fullfoldername = os.path.join(apogee_env(), 'dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/')
# Check if directory exists
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
filename = 'allStar-l31c.2.fits'
fullfilename = os.path.join(fullfoldername, filename)
url = f'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/{filename}'
elif dr == 16:
file_hash = '4d191dede9048046e31429bc1890ab3f241739f5'
fullfoldername = os.path.join(apogee_env(), 'apogeework/apogee/spectro/aspcap/r10/l31c/')
# Check if directory exists
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
filename = 'allStar-r10-l31c-58297.fits'
fullfilename = os.path.join(fullfoldername, filename)
url = f'https://data.sdss.org/sas/apogeework/apogee/spectro/aspcap/r10/l31c/{filename}'
else:
raise ValueError('allstar() only supports APOGEE DR13-DR16')
# check file integrity
if os.path.isfile(fullfilename) and flag is None:
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print('File corruption detected, astroNN attempting to download again')
allstar(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
# Check if files exists
if not os.path.isfile(os.path.join(fullfoldername, filename)) or flag == 1:
with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:
try:
urllib.request.urlretrieve(url, fullfilename, reporthook=t.update_to)
print(f'Downloaded DR{dr:d} allStar file catalog successfully to {fullfilename}')
checksum = sha1_checksum(fullfilename)
if checksum != file_hash.lower():
print('File corruption detected, astroNN attempting to download again')
allstar(dr=dr, flag=1)
except urllib.request.HTTPError as emsg:
if '401' in str(emsg):
fullfilename = __apogee_credentials_downloader(url, fullfilename)
elif '404' in str(emsg):
print(f'{url} cannot be found on server, skipped')
fullfilename = warning_flag
else:
print(f"Unknown error occurred - {emsg}")
fullfilename = warning_flag
return fullfilename
def visit_spectra(dr=None, location=None, apogee=None, telescope=None, verbose=1, flag=None, commission=False):
"""
Download the required individual spectra file a.k.a apStar
:param dr: APOGEE DR
:type dr: int
:param location: Location ID [Optional]
:type location: int
:param apogee: Apogee ID
:type apogee: str
:param telescope: Telescope ID, for example 'apo25m' or 'lco25m'
:type telescope: str
:param verbose: verbose
:type verbose: int
:param flag: 0: normal, 1: force to re-download
:type flag: int
:param commission: whether the spectra is taken during commissioning
:type commission: bool
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History:
| 2017-Nov-11 - Written - Henry Leung (University of Toronto)
| 2018-Aug-31 - Updated - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if location is None: # for DR16=<, location is expected to be none because field is used
global _ALLSTAR_TEMP
if not str(f'dr{dr}') in _ALLSTAR_TEMP:
_ALLSTAR_TEMP[f'dr{dr}'] = fits.getdata(allstar(dr=dr))
if telescope is None:
matched_idx = [np.nonzero(_ALLSTAR_TEMP[f'dr{dr}']['APOGEE_ID'] == apogee)[0]][0]
else:
matched_idx = [np.nonzero([(_ALLSTAR_TEMP[f'dr{dr}']['APOGEE_ID'] == apogee) &
(_ALLSTAR_TEMP[f'dr{dr}']['TELESCOPE'] == telescope)])][0][1]
if len(matched_idx) == 0:
raise ValueError(f"No entry found in allstar DR{dr} met with your requirement!!")
location = _ALLSTAR_TEMP[f'dr{dr}']['LOCATION_ID'][matched_idx][0]
field = _ALLSTAR_TEMP[f'dr{dr}']['FIELD'][matched_idx][0]
telescope = _ALLSTAR_TEMP[f'dr{dr}']['TELESCOPE'][matched_idx][0]
if dr == 13:
reduce_prefix = 'r6'
str1 = f'https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/{location}/'
if commission:
filename = f'apStarC-{reduce_prefix}-{apogee}.fits'
else:
filename = f'apStar-{reduce_prefix}-{apogee}.fits'
urlstr = str1 + filename
hash_filename = f'{reduce_prefix}_stars_apo25m_{location}.sha1sum'
fullfoldername = os.path.join(apogee_env(), f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/',
str(location))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
# check hash file
full_hash_filename = os.path.join(fullfoldername, hash_filename)
if not os.path.isfile(full_hash_filename):
# return warning flag if the location_id cannot even be found
try:
urllib.request.urlopen(str1)
except urllib.request.HTTPError:
return warning_flag
urllib.request.urlretrieve(str1 + hash_filename, full_hash_filename)
hash_list = np.loadtxt(full_hash_filename, dtype='str').T
fullfilename = os.path.join(apogee_env(), f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/',
str(location), filename)
elif dr == 14:
reduce_prefix = 'r8'
str1 = f'https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/{location}/'
if commission:
filename = f'apStarC-{reduce_prefix}-{apogee}.fits'
else:
filename = f'apStar-{reduce_prefix}-{apogee}.fits'
urlstr = str1 + filename
hash_filename = f'{reduce_prefix}_stars_apo25m_{location}.sha1sum'
fullfoldername = os.path.join(apogee_env(), f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/',
str(location))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
# check hash file
full_hash_filename = os.path.join(fullfoldername, hash_filename)
if not os.path.isfile(full_hash_filename):
# return warning flag if the location_id cannot even be found
try:
urllib.request.urlopen(str1)
except urllib.request.HTTPError:
return warning_flag
urllib.request.urlretrieve(str1 + hash_filename, full_hash_filename)
hash_list = np.loadtxt(full_hash_filename, dtype='str').T
fullfilename = os.path.join(apogee_env(), f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/',
str(location), filename)
elif dr == 16:
reduce_prefix = 'r10'
str1 = f'https://data.sdss.org/sas/apogeework/apogee/spectro/redux/{reduce_prefix}/stars/{telescope}/{field}/'
if commission:
filename = f'apStarC-{reduce_prefix}-{apogee}.fits'
else:
filename = f'apStar-{reduce_prefix}-{apogee}.fits'
urlstr = str1 + filename
hash_filename = f'{reduce_prefix}_stars_apo25m_{location}.sha1sum'
fullfoldername = os.path.join(apogee_env(),
f'apogeework/apogee/spectro/redux/{reduce_prefix}/stars/{telescope}/',
str(f'{field}'))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
# # check hash file
# full_hash_filename = os.path.join(fullfoldername, hash_filename)
# if not os.path.isfile(full_hash_filename):
# # return warning flag if the location_id cannot even be found
# try:
# urllib.request.urlopen(str1)
# except urllib.request.HTTPError:
# return warning_flag
# urllib.request.urlretrieve(str1 + hash_filename, full_hash_filename)
#
# hash_list = np.loadtxt(full_hash_filename, dtype='str').T
hash_list = np.array([np.array(['yyy', 'yyy', 'yyy']), np.array(['zzz', 'zzz', 'zzz'])])
fullfilename = os.path.join(apogee_env(),
f'apogeework/apogee/spectro/redux/{reduce_prefix}/stars/{telescope}/',
str(f'{field}'), filename)
else:
raise ValueError('visit_spectra() only supports DR13-DR16')
# In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too
# visit spectra has a different filename in checksum
# handle the case where apogee_id cannot be found
hash_idx = [i for i, item in enumerate(hash_list[1]) if f'apStar-{reduce_prefix}-{apogee}' in item]
file_hash = hash_list[0][hash_idx]
if os.path.isfile(fullfilename) and flag is None:
checksum = sha1_checksum(fullfilename)
if checksum != file_hash and len(file_hash) != 0:
print('File corruption detected, astroNN attempting to download again')
visit_spectra(dr=dr, location=location, apogee=apogee, verbose=verbose, flag=1)
if verbose:
print(fullfilename + ' was found!')
elif not os.path.isfile(fullfilename) or flag == 1:
try:
urllib.request.urlretrieve(urlstr, fullfilename)
print(f'Downloaded DR{dr} individual visit file successfully to {fullfilename}')
checksum = sha1_checksum(fullfilename)
if checksum != file_hash and len(file_hash) != 0:
print('File corruption detected, astroNN attempting to download again')
visit_spectra(dr=dr, location=location, apogee=apogee, verbose=verbose, flag=1)
except urllib.request.HTTPError as emsg:
if '401' in str(emsg):
fullfilename = __apogee_credentials_downloader(urlstr, fullfilename)
elif '404' in str(emsg):
print(f'{urlstr} cannot be found on server, skipped')
fullfilename = warning_flag
else:
print(f"Unknown error occurred - {emsg}")
fullfilename = warning_flag
return fullfilename
def combined_spectra(dr=None,
location=None,
apogee=None,
telescope=None,
verbose=1,
flag=None):
"""
Download the required combined spectra file a.k.a aspcapStar
:param dr: APOGEE DR
:type dr: int
:param location: Location ID [Optional]
:type location: int
:param apogee: Apogee ID
:type apogee: str
:param telescope: Telescope ID, for example 'apo25m' or 'lco25m'
:type telescope: str
:param flag: 0: normal, 1: force to re-download
:type flag: int
:return: full file path and download in background if not found locally, False if cannot be found on server
:rtype: str
:History:
| 2017-Oct-15 - Written - Henry Leung (University of Toronto)
| 2018-Aug-31 - Updated - Henry Leung (University of Toronto)
"""
dr = apogee_default_dr(dr=dr)
if location is None:
global _ALLSTAR_TEMP
if not str(f'dr{dr}') in _ALLSTAR_TEMP:
_ALLSTAR_TEMP[f'dr{dr}'] = fits.getdata(allstar(dr=dr))
if telescope is None:
matched_idx = [
np.nonzero(_ALLSTAR_TEMP[f'dr{dr}']['APOGEE_ID'] == apogee)[0]
][0]
else:
matched_idx = [
np.nonzero([
(_ALLSTAR_TEMP[f'dr{dr}']['APOGEE_ID'] == apogee) &
(_ALLSTAR_TEMP[f'dr{dr}']['TELESCOPE'] == telescope)
])
][0][1]
if len(matched_idx) == 0:
raise ValueError(
f"No entry found in allstar DR{dr} met with your requirement!!"
)
location = _ALLSTAR_TEMP[f'dr{dr}']['LOCATION_ID'][matched_idx][0]
field = _ALLSTAR_TEMP[f'dr{dr}']['FIELD'][matched_idx][0]
telescope = _ALLSTAR_TEMP[f'dr{dr}']['TELESCOPE'][matched_idx][0]
if dr == 13:
reduce_prefix = 'r6'
aspcap_code = 'l30e'
str1 = f'https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/{location}/'
filename = f'aspcapStar-{reduce_prefix}-{aspcap_code}.2-{apogee}.fits'
hash_filename = f'stars_{aspcap_code}_{aspcap_code}.2_{location}.sha1sum'
urlstr = str1 + filename
# check folder existence
fullfoldername = os.path.join(
apogee_env(),
f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/',
str(location))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(
apogee_env(),
f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/',
str(location), filename)
elif dr == 14:
reduce_prefix = 'r8'
aspcap_code = 'l31c'
str1 = f'https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/{location}/'
filename = f'aspcapStar-{reduce_prefix}-{aspcap_code}.2-{apogee}.fits'
hash_filename = f'stars_{aspcap_code}_{aspcap_code}.2_{location}.sha1sum'
urlstr = str1 + filename
# check folder existence
fullfoldername = os.path.join(
apogee_env(),
f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/',
str(location))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(
apogee_env(),
f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/',
str(location), filename)
elif dr == 15:
reduce_prefix = 't9'
aspcap_code = 'l31c'
str1 = f'https://data.sdss.org/sas/apogeework/apogee/spectro/aspcap/{reduce_prefix}/{aspcap_code}/{telescope}/{field}/'
filename = f'aspcapStar-{reduce_prefix}-{apogee}.fits'
hash_filename = f'stars_{reduce_prefix}_{reduce_prefix}.2_{location}.sha1sum'
urlstr = str1 + filename
# check folder existence
fullfoldername = os.path.join(
apogee_env(),
f'apogeework/apogee/spectro/aspcap/{reduce_prefix}/{aspcap_code}/{telescope}',
str(f'{field}'))
if not os.path.exists(fullfoldername):
os.makedirs(fullfoldername)
fullfilename = os.path.join(
apogee_env(),
f'apogeework/apogee/spectro/aspcap/{reduce_prefix}/{aspcap_code}/{telescope}',
str(f'{field}'), filename)
else:
raise ValueError(
'combined_spectra() only supports DR13, DR14 and DR15')
# check hash file
if dr != 15:
full_hash_filename = os.path.join(fullfoldername, hash_filename)
if not os.path.isfile(full_hash_filename):
# return warning flag if the location_id cannot even be found
try:
urllib.request.urlopen(str1)
except urllib.request.HTTPError:
return warning_flag
urllib.request.urlretrieve(str1 + hash_filename,
full_hash_filename)
hash_list = np.loadtxt(full_hash_filename, dtype='str').T
else:
hash_list = np.array(
[np.array(['yyy', 'yyy', 'yyy']),
np.array(['zzz', 'zzz', 'zzz'])])
# In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too
file_hash = hash_list[0][np.argwhere(hash_list[1] == filename)]
if os.path.isfile(fullfilename) and flag is None:
checksum = sha1_checksum(fullfilename)
if checksum != file_hash and len(file_hash) != 0:
print(
'File corruption detected, astroNN attempting to download again'
)
combined_spectra(dr=dr,
location=location,
apogee=apogee,
verbose=verbose,
flag=1)
if verbose == 1:
print(fullfilename + ' was found!')
elif not os.path.isfile(fullfilename) or flag == 1:
try:
urllib.request.urlretrieve(urlstr, fullfilename)
print(
f'Downloaded DR14 combined file successfully to {fullfilename}'
)
checksum = sha1_checksum(fullfilename)
if checksum != file_hash and len(file_hash) != 0:
print(
'File corruption detected, astroNN attempting to download again'
)
combined_spectra(dr=dr,
location=location,
apogee=apogee,
verbose=verbose,
flag=1)
except urllib.request.HTTPError as emsg:
if '401' in str(emsg):
__apogee_credentials_downloader(urlstr, fullfilename)
elif '404' in str(emsg):
print(f'{urlstr} cannot be found on server, skipped')
fullfilename = warning_flag
else:
print(f"Unknown error occured - {emsg}")
fullfilename = warning_flag
return fullfilename
