def allvisit(dr=None, flag=None): """ Download the allVisit file (catalog of properties from individual visit spectra) :param dr: APOGEE DR :type dr: int :param flag: 0: normal, 1: force to re-download :type flag: int :return: full file path and download in background if not found locally, False if cannot be found on server :rtype: str :History: 2017-Oct-11 - Written - Henry Leung (University of Toronto) """ dr = apogee_default_dr(dr=dr) if dr == 13: file_hash = '2a3b13ccd40a2c8aea8321be9630117922d55b51' # Check if directory exists fullfilepath = os.path.join(apogee_env(), 'dr13/apogee/spectro/redux/r6/') if not os.path.exists(fullfilepath): os.makedirs(fullfilepath) filename = 'allVisit-l30e.2.fits' fullfilename = os.path.join(fullfilepath, filename) url = f'https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/{filename}' elif dr == 14: file_hash = 'abcecbcdc5fe8d00779738702c115633811e6bbd' # Check if directory exists fullfilepath = os.path.join(apogee_env(), 'dr14/apogee/spectro/redux/r8/') if not os.path.exists(fullfilepath): os.makedirs(fullfilepath) filename = 'allVisit-l31c.2.fits' fullfilename = os.path.join(fullfilepath, filename) url = f'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/{filename}' else: raise ValueError('allvisit() only supports APOGEE DR13-DR15') # check file integrity if os.path.isfile(fullfilename) and flag is None: checksum = sha1_checksum(fullfilename) if checksum != file_hash.lower(): print('File corruption detected, astroNN attempting to download again') allvisit(dr=dr, flag=1) else: print(fullfilename + ' was found!') elif not os.path.isfile(os.path.join(fullfilepath, filename)) or flag == 1: with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t: urllib.request.urlretrieve(url, fullfilename, reporthook=t.update_to) print(f'Downloaded DR{dr:d} allVisit file catalog successfully to {fullfilepath}') checksum = sha1_checksum(fullfilename) if checksum != file_hash.lower(): print('File corruption detected, astroNN attempting to download again') allstar(dr=dr, flag=1) return fullfilename
def apogee_distances(dr=None, flag=None): """ Download the Apogee Distances catalogue :param dr: Apogee DR :type dr: int :param flag: Force to download if flag=1 :type flag: int :return: full file path :rtype: str :History: 2018-Jan-24 - Written - Henry Leung (University of Toronto) """ dr = apogee_default_dr(dr=dr) if dr == 14: file_hash = 'b33c8419be784b1be3d14af3ee9696c6ac31830f' str1 = 'https://data.sdss.org/sas/dr14/apogee/vac/apogee-distances/' filename = f'apogee_distances-DR{dr}.fits' urlstr = str1 + filename fullfilename = os.path.join(apogee_env(), 'dr14/apogee/vac/apogee-distances/') if not os.path.exists(fullfilename): os.makedirs(fullfilename) fullfilename = os.path.join(apogee_env(), 'dr14/apogee/vac/apogee-distances/', filename) else: raise ValueError('apogee_distances() only supports DR14') # check file integrity if os.path.isfile(fullfilename) and flag is None: checksum = sha1_checksum(fullfilename) if checksum != file_hash.lower(): print('File corruption detected, astroNN attempting to download again') apogee_distances(dr=dr, flag=1) else: print(fullfilename + ' was found!') elif not os.path.isfile(fullfilename) or flag == 1: try: with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=urlstr.split('/')[-1]) as t: urllib.request.urlretrieve(urlstr, fullfilename, reporthook=t.update_to) print(f'Downloaded DR{dr} Distances successfully to {fullfilename}') checksum = sha1_checksum(fullfilename) if checksum != file_hash.lower(): print('File corruption detected, astroNN attempting to download again') apogee_distances(dr=dr, flag=1) except urllib.request.HTTPError: print(f'{urlstr} cannot be found on server, skipped') fullfilename = warning_flag return fullfilename
def allstarcannon(dr=None, flag=None): """ Download the allStarCannon file (catalog of Cannon stellar parameters and abundances from combined spectra) :param dr: APOGEE DR :type dr: int :param flag: 0: normal, 1: force to re-download :type flag: int :return: full file path and download in background if not found locally, False if cannot be found on server :rtype: str :History: 2017-Oct-24 - Written - Henry Leung (University of Toronto) """ dr = apogee_default_dr(dr=dr) if dr == 14: # Check if directory exists fullfoldername = os.path.join(apogee_env(), 'dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/cannon/') # Check if directory exists if not os.path.exists(fullfoldername): os.makedirs(fullfoldername) filename = 'allStarCannon-l31c.2.fits' fullfilename = os.path.join(fullfoldername, filename) file_hash = '64d485e95b3504df0b795ab604e21a71d5c7ae45' url = f'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/cannon/{filename}' else: raise ValueError('allstarcannon() only supports APOGEE DR14-DR15') # check file integrity if os.path.isfile(fullfilename) and flag is None: checksum = sha1_checksum(fullfilename) if checksum != file_hash.lower(): print('File corruption detected, astroNN attempting to download again') allstarcannon(dr=dr, flag=1) else: print(fullfilename + ' was found!') # Check if files exists if not os.path.isfile(os.path.join(fullfoldername, filename)) or flag == 1: with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t: urllib.request.urlretrieve(url, fullfilename, reporthook=t.update_to) print(f'Downloaded DR{dr:d} allStarCannon file catalog successfully to {fullfilename}') checksum = sha1_checksum(fullfilename) if checksum != file_hash.lower(): print('File corruption detected, astroNN attempting to download again') allstarcannon(dr=dr, flag=1) return fullfilename
def test_checksum(self): import astroNN from astroNN.shared.downloader_tools import md5_checksum, sha1_checksum, sha256_checksum anderson2017_path = os.path.join(os.path.dirname(astroNN.__path__[0]), 'astroNN', 'data', 'anderson_2017_dr14_parallax.npz') md5_pred = md5_checksum(anderson2017_path) sha1_pred = sha1_checksum(anderson2017_path) sha256_pred = sha256_checksum(anderson2017_path) # read answer hashed by Windows Get-FileHash self.assertEqual(md5_pred, '9C714F5FE22BB7C4FF9EA32F3E859D73'.lower()) self.assertEqual(sha1_pred, '733C0227CF93DB0CD6106B5349402F251E7ED735'.lower()) self.assertEqual(sha256_pred, '36C265C907F440114D747DA21D2A014D32B5E442D541F183C0EE862F5865FD26'.lower())
def apogee_vac_rc(dr=None, flag=None): """ NAME: apogee_vac_rc PURPOSE: download the red clumps catalogue INPUT: dr (int): APOGEE DR, example dr=14 flag (int): 0: normal, 1: force to re-download OUTPUT: (path): full file path and download in background HISTORY: 2017-Nov-16 - Written - Henry Leung (University of Toronto) """ dr = apogee_default_dr(dr=dr) if dr == 13: file_hash = '5e87eb3ba202f9db24216978dafb19d39d382fc6' str1 = 'https://data.sdss.org/sas/dr13/apogee/vac/apogee-rc/cat/' filename = f'apogee-rc-DR{dr}.fits' urlstr = str1 + filename fullfilename = os.path.join(apogee_env(), 'dr13/apogee/vac/apogee-rc/cat/') if not os.path.exists(fullfilename): os.makedirs(fullfilename) fullfilename = os.path.join(apogee_env(), 'dr13/apogee/vac/apogee-rc/cat/', filename) elif dr == 14: file_hash = '104513070f1c280954f3d1886cac429dbdf2eaf6' str1 = 'https://data.sdss.org/sas/dr14/apogee/vac/apogee-rc/cat/' filename = f'apogee-rc-DR{dr}.fits' urlstr = str1 + filename fullfilename = os.path.join(apogee_env(), 'dr14/apogee/vac/apogee-rc/cat/') if not os.path.exists(fullfilename): os.makedirs(fullfilename) fullfilename = os.path.join(apogee_env(), 'dr14/apogee/vac/apogee-rc/cat/', filename) else: raise ValueError('apogee_vac_rc() only supports DR13 or DR14') # check file integrity if os.path.isfile(fullfilename) and flag is None: checksum = sha1_checksum(fullfilename) if checksum != file_hash.lower(): print( 'File corruption detected, astroNN attempting to download again' ) apogee_vac_rc(dr=dr, flag=1) else: print(fullfilename + ' was found!') elif not os.path.isfile(fullfilename) or flag == 1: try: with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=urlstr.split('/')[-1]) as t: urllib.request.urlretrieve(urlstr, fullfilename, reporthook=t.update_to) print( f'Downloaded DR{dr} Red Clumps Catalog successfully to {fullfilename}' ) checksum = sha1_checksum(fullfilename) if checksum != file_hash.lower(): print( 'File corruption detected, astroNN attempting to download again' ) apogee_vac_rc(dr=dr, flag=1) except urllib.request.HTTPError: print(f'{urlstr} cannot be found on server, skipped') fullfilename = warning_flag return fullfilename
def visit_spectra(dr=None, location=None, apogee=None, verbose=1, flag=None): """ Download the required individual spectra file :param dr: APOGEE DR :type dr: int :param location: Location ID [Optional] :type location: int :param apogee: Apogee ID :type apogee: str :param flag: 0: normal, 1: force to re-download :type flag: int :return: full file path and download in background if not found locally, False if cannot be found on server :rtype: str :History: 2017-Nov-11 - Written - Henry Leung (University of Toronto) """ dr = apogee_default_dr(dr=dr) if location is None: global _ALLSTAR_TEMP if _ALLSTAR_TEMP is None: _ALLSTAR_TEMP = fits.getdata(allstar(dr=dr)) location = _ALLSTAR_TEMP['LOCATION_ID'][np.nonzero( _ALLSTAR_TEMP['APOGEE_ID'] == apogee)][0] if dr == 13: reduce_prefix = 'r6' str1 = f'https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/stars/apo25m/{location}/' filename = f'apStar-r6-{apogee}.fits' urlstr = str1 + filename hash_filename = f'r6_stars_apo25m_{location}.sha1sum' fullfoldername = os.path.join( apogee_env(), 'dr13/apogee/spectro/redux/r6/stars/apo25m/', str(location)) if not os.path.exists(fullfoldername): os.makedirs(fullfoldername) # check hash file full_hash_filename = os.path.join(fullfoldername, hash_filename) if not os.path.isfile(full_hash_filename): # return warning flag if the location_id cannot even be found try: urllib.request.urlopen(str1) except urllib.request.HTTPError: return warning_flag urllib.request.urlretrieve(str1 + hash_filename, full_hash_filename) hash_list = np.loadtxt(full_hash_filename, dtype='str').T fullfilename = os.path.join( apogee_env(), 'dr13/apogee/spectro/redux/r6/stars/apo25m/', str(location), filename) elif dr == 14: reduce_prefix = 'r8' str1 = f'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/stars/apo25m/{location}/' filename = f'apStar-r8-{apogee}.fits' urlstr = str1 + filename hash_filename = f'r8_stars_apo25m_{location}.sha1sum' fullfoldername = os.path.join( apogee_env(), 'dr14/apogee/spectro/redux/r8/stars/apo25m/', str(location)) if not os.path.exists(fullfoldername): os.makedirs(fullfoldername) # check hash file full_hash_filename = os.path.join(fullfoldername, hash_filename) if not os.path.isfile(full_hash_filename): # return warning flag if the location_id cannot even be found try: urllib.request.urlopen(str1) except urllib.request.HTTPError: return warning_flag urllib.request.urlretrieve(str1 + hash_filename, full_hash_filename) hash_list = np.loadtxt(full_hash_filename, dtype='str').T fullfilename = os.path.join( apogee_env(), 'dr14/apogee/spectro/redux/r8/stars/apo25m/', str(location), filename) else: raise ValueError('visit_spectra() only supports DR13 or DR14') # In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too # visit spectra has a different filename in checksum # handle the case where apogee_id cannot be found hash_idx = [ i for i, item in enumerate(hash_list[1]) if f'apStar-{reduce_prefix}-{apogee}' in item ] file_hash = hash_list[0][hash_idx] if os.path.isfile(fullfilename) and flag is None: checksum = sha1_checksum(fullfilename) if checksum != file_hash and len(file_hash) != 0: print( 'File corruption detected, astroNN attempting to download again' ) visit_spectra(dr=dr, location=location, apogee=apogee, verbose=verbose, flag=1) if verbose == 1: print(fullfilename + ' was found!') elif not os.path.isfile(fullfilename) or flag == 1: try: urllib.request.urlretrieve(urlstr, fullfilename) print( f'Downloaded DR14 individual visit file successfully to {fullfilename}' ) checksum = sha1_checksum(fullfilename) if checksum != file_hash and len(file_hash) != 0: print( 'File corruption detected, astroNN attempting to download again' ) visit_spectra(dr=dr, location=location, apogee=apogee, verbose=verbose, flag=1) except urllib.request.HTTPError: print(f'{urlstr} cannot be found on server, skipped') fullfilename = warning_flag return fullfilename
def allstar(dr=None, flag=None): """ NAME: allstar PURPOSE: download the allStar file (catalog of ASPCAP stellar parameters and abundances from combined spectra) INPUT: dr (int): APOGEE DR, example dr=14 flag (int): 0: normal, 1: force to re-download OUTPUT: (path): full file path and download in background HISTORY: 2017-Oct-09 - Written - Henry Leung (University of Toronto) """ dr = apogee_default_dr(dr=dr) if dr == 13: file_hash = '1718723ada3018de94e1022cd57d4d950a74f91f' # Check if directory exists fullfoldername = os.path.join( apogee_env(), 'dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/') if not os.path.exists(fullfoldername): os.makedirs(fullfoldername) filename = 'allStar-l30e.2.fits' fullfilename = os.path.join(fullfoldername, filename) url = f'https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/{filename}' elif dr == 14: file_hash = 'a7e1801924661954da792e377ad54f412219b105' fullfoldername = os.path.join( apogee_env(), 'dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/') # Check if directory exists if not os.path.exists(fullfoldername): os.makedirs(fullfoldername) filename = 'allStar-l31c.2.fits' fullfilename = os.path.join(fullfoldername, filename) url = f'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/{filename}' else: raise ValueError('allstar() only supports APOGEE DR13 and DR14') # check file integrity if os.path.isfile(fullfilename) and flag is None: checksum = sha1_checksum(fullfilename) if checksum != file_hash.lower(): print( 'File corruption detected, astroNN attempting to download again' ) allstar(dr=dr, flag=1) else: print(fullfilename + ' was found!') # Check if files exists if not os.path.isfile(os.path.join(fullfoldername, filename)) or flag == 1: with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t: urllib.request.urlretrieve(url, fullfilename, reporthook=t.update_to) print( f'Downloaded DR{dr:d} allStar file catalog successfully to {fullfilename}' ) checksum = sha1_checksum(fullfilename) if checksum != file_hash.lower(): print( 'File corruption detected, astroNN attempting to download again' ) allstar(dr=dr, flag=1) return fullfilename
def allstar(dr=None, flag=None): """ Download the allStar file (catalog of ASPCAP stellar parameters and abundances from combined spectra) :param dr: APOGEE DR :type dr: int :param flag: 0: normal, 1: force to re-download :type flag: int :return: full file path and download in background if not found locally, False if cannot be found on server :rtype: str :History: 2017-Oct-09 - Written - Henry Leung (University of Toronto) """ dr = apogee_default_dr(dr=dr) if dr == 13: file_hash = '1718723ada3018de94e1022cd57d4d950a74f91f' # Check if directory exists fullfoldername = os.path.join(apogee_env(), 'dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/') if not os.path.exists(fullfoldername): os.makedirs(fullfoldername) filename = 'allStar-l30e.2.fits' fullfilename = os.path.join(fullfoldername, filename) url = f'https://data.sdss.org/sas/dr13/apogee/spectro/redux/r6/stars/l30e/l30e.2/{filename}' elif dr == 14: file_hash = 'a7e1801924661954da792e377ad54f412219b105' fullfoldername = os.path.join(apogee_env(), 'dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/') # Check if directory exists if not os.path.exists(fullfoldername): os.makedirs(fullfoldername) filename = 'allStar-l31c.2.fits' fullfilename = os.path.join(fullfoldername, filename) url = f'https://data.sdss.org/sas/dr14/apogee/spectro/redux/r8/stars/l31c/l31c.2/{filename}' elif dr == 16: file_hash = '4d191dede9048046e31429bc1890ab3f241739f5' fullfoldername = os.path.join(apogee_env(), 'apogeework/apogee/spectro/aspcap/r10/l31c/') # Check if directory exists if not os.path.exists(fullfoldername): os.makedirs(fullfoldername) filename = 'allStar-r10-l31c-58297.fits' fullfilename = os.path.join(fullfoldername, filename) url = f'https://data.sdss.org/sas/apogeework/apogee/spectro/aspcap/r10/l31c/{filename}' else: raise ValueError('allstar() only supports APOGEE DR13-DR16') # check file integrity if os.path.isfile(fullfilename) and flag is None: checksum = sha1_checksum(fullfilename) if checksum != file_hash.lower(): print('File corruption detected, astroNN attempting to download again') allstar(dr=dr, flag=1) else: print(fullfilename + ' was found!') # Check if files exists if not os.path.isfile(os.path.join(fullfoldername, filename)) or flag == 1: with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t: try: urllib.request.urlretrieve(url, fullfilename, reporthook=t.update_to) print(f'Downloaded DR{dr:d} allStar file catalog successfully to {fullfilename}') checksum = sha1_checksum(fullfilename) if checksum != file_hash.lower(): print('File corruption detected, astroNN attempting to download again') allstar(dr=dr, flag=1) except urllib.request.HTTPError as emsg: if '401' in str(emsg): fullfilename = __apogee_credentials_downloader(url, fullfilename) elif '404' in str(emsg): print(f'{url} cannot be found on server, skipped') fullfilename = warning_flag else: print(f"Unknown error occurred - {emsg}") fullfilename = warning_flag return fullfilename
def visit_spectra(dr=None, location=None, apogee=None, telescope=None, verbose=1, flag=None, commission=False): """ Download the required individual spectra file a.k.a apStar :param dr: APOGEE DR :type dr: int :param location: Location ID [Optional] :type location: int :param apogee: Apogee ID :type apogee: str :param telescope: Telescope ID, for example 'apo25m' or 'lco25m' :type telescope: str :param verbose: verbose :type verbose: int :param flag: 0: normal, 1: force to re-download :type flag: int :param commission: whether the spectra is taken during commissioning :type commission: bool :return: full file path and download in background if not found locally, False if cannot be found on server :rtype: str :History: | 2017-Nov-11 - Written - Henry Leung (University of Toronto) | 2018-Aug-31 - Updated - Henry Leung (University of Toronto) """ dr = apogee_default_dr(dr=dr) if location is None: # for DR16=<, location is expected to be none because field is used global _ALLSTAR_TEMP if not str(f'dr{dr}') in _ALLSTAR_TEMP: _ALLSTAR_TEMP[f'dr{dr}'] = fits.getdata(allstar(dr=dr)) if telescope is None: matched_idx = [np.nonzero(_ALLSTAR_TEMP[f'dr{dr}']['APOGEE_ID'] == apogee)[0]][0] else: matched_idx = [np.nonzero([(_ALLSTAR_TEMP[f'dr{dr}']['APOGEE_ID'] == apogee) & (_ALLSTAR_TEMP[f'dr{dr}']['TELESCOPE'] == telescope)])][0][1] if len(matched_idx) == 0: raise ValueError(f"No entry found in allstar DR{dr} met with your requirement!!") location = _ALLSTAR_TEMP[f'dr{dr}']['LOCATION_ID'][matched_idx][0] field = _ALLSTAR_TEMP[f'dr{dr}']['FIELD'][matched_idx][0] telescope = _ALLSTAR_TEMP[f'dr{dr}']['TELESCOPE'][matched_idx][0] if dr == 13: reduce_prefix = 'r6' str1 = f'https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/{location}/' if commission: filename = f'apStarC-{reduce_prefix}-{apogee}.fits' else: filename = f'apStar-{reduce_prefix}-{apogee}.fits' urlstr = str1 + filename hash_filename = f'{reduce_prefix}_stars_apo25m_{location}.sha1sum' fullfoldername = os.path.join(apogee_env(), f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/', str(location)) if not os.path.exists(fullfoldername): os.makedirs(fullfoldername) # check hash file full_hash_filename = os.path.join(fullfoldername, hash_filename) if not os.path.isfile(full_hash_filename): # return warning flag if the location_id cannot even be found try: urllib.request.urlopen(str1) except urllib.request.HTTPError: return warning_flag urllib.request.urlretrieve(str1 + hash_filename, full_hash_filename) hash_list = np.loadtxt(full_hash_filename, dtype='str').T fullfilename = os.path.join(apogee_env(), f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/', str(location), filename) elif dr == 14: reduce_prefix = 'r8' str1 = f'https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/{location}/' if commission: filename = f'apStarC-{reduce_prefix}-{apogee}.fits' else: filename = f'apStar-{reduce_prefix}-{apogee}.fits' urlstr = str1 + filename hash_filename = f'{reduce_prefix}_stars_apo25m_{location}.sha1sum' fullfoldername = os.path.join(apogee_env(), f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/', str(location)) if not os.path.exists(fullfoldername): os.makedirs(fullfoldername) # check hash file full_hash_filename = os.path.join(fullfoldername, hash_filename) if not os.path.isfile(full_hash_filename): # return warning flag if the location_id cannot even be found try: urllib.request.urlopen(str1) except urllib.request.HTTPError: return warning_flag urllib.request.urlretrieve(str1 + hash_filename, full_hash_filename) hash_list = np.loadtxt(full_hash_filename, dtype='str').T fullfilename = os.path.join(apogee_env(), f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/', str(location), filename) elif dr == 16: reduce_prefix = 'r10' str1 = f'https://data.sdss.org/sas/apogeework/apogee/spectro/redux/{reduce_prefix}/stars/{telescope}/{field}/' if commission: filename = f'apStarC-{reduce_prefix}-{apogee}.fits' else: filename = f'apStar-{reduce_prefix}-{apogee}.fits' urlstr = str1 + filename hash_filename = f'{reduce_prefix}_stars_apo25m_{location}.sha1sum' fullfoldername = os.path.join(apogee_env(), f'apogeework/apogee/spectro/redux/{reduce_prefix}/stars/{telescope}/', str(f'{field}')) if not os.path.exists(fullfoldername): os.makedirs(fullfoldername) # # check hash file # full_hash_filename = os.path.join(fullfoldername, hash_filename) # if not os.path.isfile(full_hash_filename): # # return warning flag if the location_id cannot even be found # try: # urllib.request.urlopen(str1) # except urllib.request.HTTPError: # return warning_flag # urllib.request.urlretrieve(str1 + hash_filename, full_hash_filename) # # hash_list = np.loadtxt(full_hash_filename, dtype='str').T hash_list = np.array([np.array(['yyy', 'yyy', 'yyy']), np.array(['zzz', 'zzz', 'zzz'])]) fullfilename = os.path.join(apogee_env(), f'apogeework/apogee/spectro/redux/{reduce_prefix}/stars/{telescope}/', str(f'{field}'), filename) else: raise ValueError('visit_spectra() only supports DR13-DR16') # In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too # visit spectra has a different filename in checksum # handle the case where apogee_id cannot be found hash_idx = [i for i, item in enumerate(hash_list[1]) if f'apStar-{reduce_prefix}-{apogee}' in item] file_hash = hash_list[0][hash_idx] if os.path.isfile(fullfilename) and flag is None: checksum = sha1_checksum(fullfilename) if checksum != file_hash and len(file_hash) != 0: print('File corruption detected, astroNN attempting to download again') visit_spectra(dr=dr, location=location, apogee=apogee, verbose=verbose, flag=1) if verbose: print(fullfilename + ' was found!') elif not os.path.isfile(fullfilename) or flag == 1: try: urllib.request.urlretrieve(urlstr, fullfilename) print(f'Downloaded DR{dr} individual visit file successfully to {fullfilename}') checksum = sha1_checksum(fullfilename) if checksum != file_hash and len(file_hash) != 0: print('File corruption detected, astroNN attempting to download again') visit_spectra(dr=dr, location=location, apogee=apogee, verbose=verbose, flag=1) except urllib.request.HTTPError as emsg: if '401' in str(emsg): fullfilename = __apogee_credentials_downloader(urlstr, fullfilename) elif '404' in str(emsg): print(f'{urlstr} cannot be found on server, skipped') fullfilename = warning_flag else: print(f"Unknown error occurred - {emsg}") fullfilename = warning_flag return fullfilename
def combined_spectra(dr=None, location=None, apogee=None, telescope=None, verbose=1, flag=None): """ Download the required combined spectra file a.k.a aspcapStar :param dr: APOGEE DR :type dr: int :param location: Location ID [Optional] :type location: int :param apogee: Apogee ID :type apogee: str :param telescope: Telescope ID, for example 'apo25m' or 'lco25m' :type telescope: str :param flag: 0: normal, 1: force to re-download :type flag: int :return: full file path and download in background if not found locally, False if cannot be found on server :rtype: str :History: | 2017-Oct-15 - Written - Henry Leung (University of Toronto) | 2018-Aug-31 - Updated - Henry Leung (University of Toronto) """ dr = apogee_default_dr(dr=dr) if location is None: global _ALLSTAR_TEMP if not str(f'dr{dr}') in _ALLSTAR_TEMP: _ALLSTAR_TEMP[f'dr{dr}'] = fits.getdata(allstar(dr=dr)) if telescope is None: matched_idx = [ np.nonzero(_ALLSTAR_TEMP[f'dr{dr}']['APOGEE_ID'] == apogee)[0] ][0] else: matched_idx = [ np.nonzero([ (_ALLSTAR_TEMP[f'dr{dr}']['APOGEE_ID'] == apogee) & (_ALLSTAR_TEMP[f'dr{dr}']['TELESCOPE'] == telescope) ]) ][0][1] if len(matched_idx) == 0: raise ValueError( f"No entry found in allstar DR{dr} met with your requirement!!" ) location = _ALLSTAR_TEMP[f'dr{dr}']['LOCATION_ID'][matched_idx][0] field = _ALLSTAR_TEMP[f'dr{dr}']['FIELD'][matched_idx][0] telescope = _ALLSTAR_TEMP[f'dr{dr}']['TELESCOPE'][matched_idx][0] if dr == 13: reduce_prefix = 'r6' aspcap_code = 'l30e' str1 = f'https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/{location}/' filename = f'aspcapStar-{reduce_prefix}-{aspcap_code}.2-{apogee}.fits' hash_filename = f'stars_{aspcap_code}_{aspcap_code}.2_{location}.sha1sum' urlstr = str1 + filename # check folder existence fullfoldername = os.path.join( apogee_env(), f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/', str(location)) if not os.path.exists(fullfoldername): os.makedirs(fullfoldername) fullfilename = os.path.join( apogee_env(), f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/', str(location), filename) elif dr == 14: reduce_prefix = 'r8' aspcap_code = 'l31c' str1 = f'https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/{location}/' filename = f'aspcapStar-{reduce_prefix}-{aspcap_code}.2-{apogee}.fits' hash_filename = f'stars_{aspcap_code}_{aspcap_code}.2_{location}.sha1sum' urlstr = str1 + filename # check folder existence fullfoldername = os.path.join( apogee_env(), f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/', str(location)) if not os.path.exists(fullfoldername): os.makedirs(fullfoldername) fullfilename = os.path.join( apogee_env(), f'dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/', str(location), filename) elif dr == 15: reduce_prefix = 't9' aspcap_code = 'l31c' str1 = f'https://data.sdss.org/sas/apogeework/apogee/spectro/aspcap/{reduce_prefix}/{aspcap_code}/{telescope}/{field}/' filename = f'aspcapStar-{reduce_prefix}-{apogee}.fits' hash_filename = f'stars_{reduce_prefix}_{reduce_prefix}.2_{location}.sha1sum' urlstr = str1 + filename # check folder existence fullfoldername = os.path.join( apogee_env(), f'apogeework/apogee/spectro/aspcap/{reduce_prefix}/{aspcap_code}/{telescope}', str(f'{field}')) if not os.path.exists(fullfoldername): os.makedirs(fullfoldername) fullfilename = os.path.join( apogee_env(), f'apogeework/apogee/spectro/aspcap/{reduce_prefix}/{aspcap_code}/{telescope}', str(f'{field}'), filename) else: raise ValueError( 'combined_spectra() only supports DR13, DR14 and DR15') # check hash file if dr != 15: full_hash_filename = os.path.join(fullfoldername, hash_filename) if not os.path.isfile(full_hash_filename): # return warning flag if the location_id cannot even be found try: urllib.request.urlopen(str1) except urllib.request.HTTPError: return warning_flag urllib.request.urlretrieve(str1 + hash_filename, full_hash_filename) hash_list = np.loadtxt(full_hash_filename, dtype='str').T else: hash_list = np.array( [np.array(['yyy', 'yyy', 'yyy']), np.array(['zzz', 'zzz', 'zzz'])]) # In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too file_hash = hash_list[0][np.argwhere(hash_list[1] == filename)] if os.path.isfile(fullfilename) and flag is None: checksum = sha1_checksum(fullfilename) if checksum != file_hash and len(file_hash) != 0: print( 'File corruption detected, astroNN attempting to download again' ) combined_spectra(dr=dr, location=location, apogee=apogee, verbose=verbose, flag=1) if verbose == 1: print(fullfilename + ' was found!') elif not os.path.isfile(fullfilename) or flag == 1: try: urllib.request.urlretrieve(urlstr, fullfilename) print( f'Downloaded DR14 combined file successfully to {fullfilename}' ) checksum = sha1_checksum(fullfilename) if checksum != file_hash and len(file_hash) != 0: print( 'File corruption detected, astroNN attempting to download again' ) combined_spectra(dr=dr, location=location, apogee=apogee, verbose=verbose, flag=1) except urllib.request.HTTPError as emsg: if '401' in str(emsg): __apogee_credentials_downloader(urlstr, fullfilename) elif '404' in str(emsg): print(f'{urlstr} cannot be found on server, skipped') fullfilename = warning_flag else: print(f"Unknown error occured - {emsg}") fullfilename = warning_flag return fullfilename