def gaia_source(dr=None): # TODO not working """ NAME: gaia_source PURPOSE: download the gaia_source files INPUT: OUTPUT: (just downloads) HISTORY: 2017-Oct-13 Henry Leung """ dr = gaia_default_dr(dr=dr) if dr == 1: for j in range(0, 20, 1): for i in range(0, 256, 1): urlstr = 'http://cdn.gea.esac.esa.int/Gaia/gaia_source/fits/GaiaSource_000-0{:02d}-{:03d}.fits'.format( j, i) with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=urlstr.split('/')[-1]) as t: urllib.request.urlretrieve(urlstr, reporthook=t.update_to) print('Downloaded Gaia DR{:d} Gaia Source ({:d} of {:d}) file catalog successfully to {}') % ( dr, (j * 256 + i), 256 * 20 + 112, currentdir) for i in range(0, 111, 1): urlstr = 'http://cdn.gea.esac.esa.int/Gaia/gaia_source/fits/GaiaSource_000-020-{:03d}.fits'.format(i) with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=urlstr.split('/')[-1]) as t: urllib.request.urlretrieve(urlstr, reporthook=t.update_to) print('Downloaded Gaia DR{:d} Gaia Source ({:d} of {:d}) file catalog successfully to {}') % ( dr, (20 * 256 + i), 256 * 20 + 112, currentdir) else: raise ValueError('[astroNN.gaia.downloader.gaia_source()] only supports Gaia DR1 Gaia Source') return None
def tgas(dr=None): """ NAME: tgas PURPOSE: download the tgas files INPUT: OUTPUT: (just downloads) HISTORY: 2017-Oct-13 Henry Leung """ # Check if dr arguement is provided, if none then use default dr = gaia_default_dr(dr=dr) fulllist = [] if dr == 1: # Check if directory exists folderpath =os.path.join(_GAIA_DATA, 'Gaia/tgas_source/fits/') if not os.path.exists(folderpath): os.makedirs(folderpath) for i in range(0, 16, 1): filename = 'TgasSource_000-000-0{:02d}.fits'.format(i) fullfilename = os.path.join(folderpath, filename) urlstr = 'http://cdn.gea.esac.esa.int/Gaia/tgas_source/fits/{}'.format(filename) # Check if files exists if not os.path.isfile(fullfilename): # progress bar with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=urlstr.split('/')[-1]) as t: # Download urllib.request.urlretrieve(urlstr, fullfilename, reporthook=t.update_to) print('Downloaded Gaia DR{:d} TGAS ({:d} of 15) file catalog successfully to {}'.format(dr, i, fullfilename)) else: print(fullfilename + ' was found!') fulllist.extend([fullfilename]) else: raise ValueError('[astroNN.gaia.downloader.tgas()] only supports Gaia DR1 TGAS') return fulllist
def gaia_source(dr=None, flag=None): """ NAME: gaia_source PURPOSE: download the gaia_source files INPUT: dr (int): Gaia DR, example dr=1 flag (int): 0: normal, 1: force to re-download OUTPUT: list of file path HISTORY: 2017-Oct-13 - Written - Henry Leung (University of Toronto) 2017-Nov-26 - Update - Henry Leung (University of Toronto) """ dr = gaia_default_dr(dr=dr) fulllist = [] if dr == 1: # Check if directory exists folderpath = os.path.join(gaia_env(), 'Gaia/gdr1/gaia_source/fits/') urlbase = 'http://cdn.gea.esac.esa.int/Gaia/gdr1/gaia_source/fits/' if not os.path.exists(folderpath): os.makedirs(folderpath) hash_filename = 'MD5SUM.txt' full_hash_filename = os.path.join(folderpath, hash_filename) if not os.path.isfile(full_hash_filename): urllib.request.urlretrieve(urlbase + hash_filename, full_hash_filename) hash_list = np.loadtxt(full_hash_filename, dtype='str').T for j in range(0, 20, 1): for i in range(0, 256, 1): filename = f'GaiaSource_000-0{j:0{2}d}-{i:0{3}d}.fits' urlstr = urlbase + filename fullfilename = os.path.join(folderpath, filename) file_hash = (hash_list[0])[np.argwhere( hash_list[1] == filename)] # Check if files exists if os.path.isfile(fullfilename) and flag is None: checksum = md5_checksum(fullfilename) # In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too if checksum != file_hash and len(file_hash) != 0: print(checksum) print(file_hash) print( 'File corruption detected, astroNN attempting to download again' ) gaia_source(dr=dr, flag=1) else: print(fullfilename + ' was found!') elif not os.path.isfile(fullfilename) or flag == 1: # progress bar with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=urlstr.split('/')[-1]) as t: urllib.request.urlretrieve(urlstr, fullfilename, reporthook=t.update_to) checksum = md5_checksum(fullfilename) if checksum != file_hash and len(file_hash) != 0: print( 'File corruption detected, astroNN attempting to download again' ) gaia_source(dr=dr, flag=1) print( f'Downloaded Gaia DR{dr} Gaia Source ({(j * 256 + i):d} of {(256 * 20 + 112):d}) ' f'file catalog successfully to {fullfilename}') fulllist.extend([fullfilename]) for i in range(0, 111, 1): filename = f'GaiaSource_000-020-{i:0{3}d}.fits' urlstr = urlbase + filename fullfilename = os.path.join(folderpath, filename) file_hash = (hash_list[0])[np.argwhere(hash_list[1] == filename)] # Check if files exists if os.path.isfile(fullfilename) and flag is None: checksum = md5_checksum(fullfilename) # In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too if checksum != file_hash and len(file_hash) != 0: print(checksum) print(file_hash) print( 'File corruption detected, astroNN attempting to download again' ) gaia_source(dr=dr, flag=1) else: print(fullfilename + ' was found!') elif not os.path.isfile(fullfilename) or flag == 1: # progress bar with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=urlstr.split('/')[-1]) as t: urllib.request.urlretrieve(urlstr, fullfilename, reporthook=t.update_to) checksum = md5_checksum(fullfilename) if checksum != file_hash and len(file_hash) != 0: print( 'File corruption detected, astroNN attempting to download again' ) gaia_source(dr=dr, flag=1) print( f'Downloaded Gaia DR{dr} Gaia Source ({(20 * 256 + i):d} of {(256 * 20 + 112):d}) file ' f'catalog successfully to {fullfilename}') fulllist.extend([fullfilename]) else: raise ValueError('gaia_source() only supports Gaia DR1 Gaia Source') return fulllist