def tgas(flag=None):
"""
Get path to the Gaia TGAS DR1 files, download if files not found
:return: List of file path
:rtype: list
:History: 2017-Oct-13 - Written - Henry Leung (University of Toronto)
"""
# Check if dr arguement is provided, if none then use default
fulllist = []
# Check if directory exists
folderpath = os.path.join(gaia_env(), 'Gaia/gdr1/tgas_source/fits/')
urlbase = 'http://cdn.gea.esac.esa.int/Gaia/gdr1/tgas_source/fits/'
if not os.path.exists(folderpath):
os.makedirs(folderpath)
hash_filename = 'MD5SUM.txt'
full_hash_filename = os.path.join(folderpath, hash_filename)
if not os.path.isfile(full_hash_filename):
urllib.request.urlretrieve(urlbase + hash_filename, full_hash_filename)
hash_list = np.loadtxt(full_hash_filename, dtype='str').T
for i in range(0, 16, 1):
filename = f'TgasSource_000-000-0{i:0{2}d}.fits'
fullfilename = os.path.join(folderpath, filename)
urlstr = urlbase + filename
file_hash = (hash_list[0])[np.argwhere(hash_list[1] == filename)]
# Check if files exists
if os.path.isfile(fullfilename) and flag is None:
checksum = filehash(fullfilename, algorithm='md5')
# In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too
if checksum != file_hash and len(file_hash) != 0:
print(checksum)
print(file_hash)
print('File corruption detected, astroNN is attempting to download again')
tgas(flag=1)
else:
print(fullfilename + ' was found!')
elif not os.path.isfile(fullfilename) or flag == 1:
# progress bar
with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=urlstr.split('/')[-1]) as t:
# Download
urllib.request.urlretrieve(urlstr, fullfilename, reporthook=t.update_to)
checksum = filehash(fullfilename, algorithm='md5')
if checksum != file_hash and len(file_hash) != 0:
print('File corruption detected, astroNN is attempting to download again')
tgas(flag=1)
print(f'Downloaded Gaia DR1 TGAS ({i:d} of 15) file catalog successfully to {fullfilename}')
fulllist.extend([fullfilename])
return fulllist
def gaia_source(dr=None, flag=None):
"""
NAME:
gaia_source
PURPOSE:
download the gaia_source files
INPUT:
dr (int): Gaia DR, example dr=1
flag (int): 0: normal, 1: force to re-download
OUTPUT:
list of file path
HISTORY:
2017-Oct-13 - Written - Henry Leung (University of Toronto)
2017-Nov-26 - Update - Henry Leung (University of Toronto)
"""
dr = gaia_default_dr(dr=dr)
fulllist = []
if dr == 1:
# Check if directory exists
folderpath = os.path.join(gaia_env(), 'Gaia/gdr1/gaia_source/fits/')
urlbase = 'http://cdn.gea.esac.esa.int/Gaia/gdr1/gaia_source/fits/'
if not os.path.exists(folderpath):
os.makedirs(folderpath)
hash_filename = 'MD5SUM.txt'
full_hash_filename = os.path.join(folderpath, hash_filename)
if not os.path.isfile(full_hash_filename):
urllib.request.urlretrieve(urlbase + hash_filename,
full_hash_filename)
hash_list = np.loadtxt(full_hash_filename, dtype='str').T
for j in range(0, 20, 1):
for i in range(0, 256, 1):
filename = f'GaiaSource_000-0{j:0{2}d}-{i:0{3}d}.fits'
urlstr = urlbase + filename
fullfilename = os.path.join(folderpath, filename)
file_hash = (hash_list[0])[np.argwhere(
hash_list[1] == filename)]
# Check if files exists
if os.path.isfile(fullfilename) and flag is None:
checksum = md5_checksum(fullfilename)
# In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too
if checksum != file_hash and len(file_hash) != 0:
print(checksum)
print(file_hash)
print(
'File corruption detected, astroNN attempting to download again'
)
gaia_source(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
elif not os.path.isfile(fullfilename) or flag == 1:
# progress bar
with TqdmUpTo(unit='B',
unit_scale=True,
miniters=1,
desc=urlstr.split('/')[-1]) as t:
urllib.request.urlretrieve(urlstr,
fullfilename,
reporthook=t.update_to)
checksum = md5_checksum(fullfilename)
if checksum != file_hash and len(file_hash) != 0:
print(
'File corruption detected, astroNN attempting to download again'
)
gaia_source(dr=dr, flag=1)
print(
f'Downloaded Gaia DR{dr} Gaia Source ({(j * 256 + i):d} of {(256 * 20 + 112):d}) '
f'file catalog successfully to {fullfilename}')
fulllist.extend([fullfilename])
for i in range(0, 111, 1):
filename = f'GaiaSource_000-020-{i:0{3}d}.fits'
urlstr = urlbase + filename
fullfilename = os.path.join(folderpath, filename)
file_hash = (hash_list[0])[np.argwhere(hash_list[1] == filename)]
# Check if files exists
if os.path.isfile(fullfilename) and flag is None:
checksum = md5_checksum(fullfilename)
# In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too
if checksum != file_hash and len(file_hash) != 0:
print(checksum)
print(file_hash)
print(
'File corruption detected, astroNN attempting to download again'
)
gaia_source(dr=dr, flag=1)
else:
print(fullfilename + ' was found!')
elif not os.path.isfile(fullfilename) or flag == 1:
# progress bar
with TqdmUpTo(unit='B',
unit_scale=True,
miniters=1,
desc=urlstr.split('/')[-1]) as t:
urllib.request.urlretrieve(urlstr,
fullfilename,
reporthook=t.update_to)
checksum = md5_checksum(fullfilename)
if checksum != file_hash and len(file_hash) != 0:
print(
'File corruption detected, astroNN attempting to download again'
)
gaia_source(dr=dr, flag=1)
print(
f'Downloaded Gaia DR{dr} Gaia Source ({(20 * 256 + i):d} of {(256 * 20 + 112):d}) file '
f'catalog successfully to {fullfilename}')
fulllist.extend([fullfilename])
else:
raise ValueError('gaia_source() only supports Gaia DR1 Gaia Source')
return fulllist
# ---------------------------------------------------------#
# astroNN.gaiatools.downloader: download gaia files
# ---------------------------------------------------------#
import os
import urllib.request
from astroNN.shared.downloader_tools import TqdmUpTo
from astroNN.gaia.gaia_shared import gaia_env, gaia_default_dr
currentdir = os.getcwd()
_GAIA_DATA = gaia_env()
def tgas(dr=None):
"""
NAME: tgas
PURPOSE: download the tgas files
INPUT:
OUTPUT: (just downloads)
HISTORY:
2017-Oct-13 Henry Leung
"""
# Check if dr arguement is provided, if none then use default
dr = gaia_default_dr(dr=dr)
fulllist = []
if dr == 1:
# Check if directory exists
folderpath =os.path.join(_GAIA_DATA, 'Gaia/tgas_source/fits/')
