Exemple #1
0
def allStar(rmcommissioning=True,
            main=False,
            exclude_star_bad=False,
            exclude_star_warn=False,
            ak=True,
            akvers='targ',
            rmnovisits=False,
            adddist=False,
            distredux=None,
            rmdups=False,
            raw=False):
    """
    NAME:
       allStar
    PURPOSE:
       read the allStar file
    INPUT:
       rmcommissioning= (default: True) if True, only use data obtained after commissioning
       main= (default: False) if True, only select stars in the main survey
       exclude_star_bad= (False) if True, remove stars with the STAR_BAD flag set in ASPCAPFLAG
       exclude_star_warn= (False) if True, remove stars with the STAR_WARN flag set in ASPCAPFLAG
       ak= (default: True) only use objects for which dereddened mags exist
       akvers= 'targ' (default) or 'wise': use target AK (AK_TARG) or AK derived from all-sky WISE (AK_WISE)
       rmnovisits= (False) if True, remove stars with no good visits (to go into the combined spectrum); shouldn't be necessary
       adddist= (default: False) add distances (DR10/11 Hayden distances, DR12 combined distances)
       distredux= (default: DR default) reduction on which the distances are based
       rmdups= (False) if True, remove duplicates (very slow)
       raw= (False) if True, just return the raw file, read w/ fitsio
    OUTPUT:
       allStar data
    HISTORY:
       2013-09-06 - Written - Bovy (IAS)
    """
    filePath = path.allStarPath()
    if not os.path.exists(filePath):
        download.allStar()
    #read allStar file
    data = fitsio.read(path.allStarPath())
    if raw: return data
    #Remove duplicates, cache
    if rmdups:
        dupsFilename = path.allStarPath().replace('.fits', '-nodups.fits')
        if os.path.exists(dupsFilename):
            data = fitsio.read(dupsFilename)
        else:
            sys.stdout.write(
                '\r' +
                "Removing duplicates (might take a while) and caching the duplicate-free file ...\r"
            )
            sys.stdout.flush()
            data = remove_duplicates(data)
            #Cache this file for subsequent use of rmdups
            fitsio.write(dupsFilename, data, clobber=True)
            sys.stdout.write('\r' + _ERASESTR + '\r')
            sys.stdout.flush()
    #Some cuts
    if rmcommissioning:
        indx = numpy.array(
            ['apogee.n.c'.encode('utf-8') in s for s in data['APSTAR_ID']])
        indx += numpy.array(
            ['apogee.s.c'.encode('utf-8') in s for s in data['APSTAR_ID']])
        data = data[True - indx]
    if rmnovisits:
        indx = numpy.array([s.strip() != '' for s in data['VISITS']])
        data = data[indx]
    if main:
        indx = mainIndx(data)
        data = data[indx]
    if akvers.lower() == 'targ':
        aktag = 'AK_TARG'
    elif akvers.lower() == 'wise':
        aktag = 'AK_WISE'
    if ak:
        data = data[True - numpy.isnan(data[aktag])]
        data = data[(data[aktag] > -50.)]
    if exclude_star_bad:
        data = data[(data['ASPCAPFLAG'] & 2**23) == 0]
    if exclude_star_warn:
        data = data[(data['ASPCAPFLAG'] & 2**7) == 0]
    #Add dereddened J, H, and Ks
    aj = data[aktag] * 2.5
    ah = data[aktag] * 1.55
    if _ESUTIL_LOADED:
        data = esutil.numpy_util.add_fields(data,
                                            [('J0', float), ('H0', float),
                                             ('K0', float)])
        data['J0'] = data['J'] - aj
        data['H0'] = data['H'] - ah
        data['K0'] = data['K'] - data[aktag]
        data['J0'][(data[aktag] <= -50.)] = -9999.9999
        data['H0'][(data[aktag] <= -50.)] = -9999.9999
        data['K0'][(data[aktag] <= -50.)] = -9999.9999
    else:
        warnings.warn(
            "Extinction-corrected J,H,K not added because esutil is not installed",
            RuntimeWarning)
    #Add distances
    if adddist and _ESUTIL_LOADED:
        dist = fitsio.read(path.distPath(), 1)
        h = esutil.htm.HTM()
        m1, m2, d12 = h.match(dist['RA'],
                              dist['DEC'],
                              data['RA'],
                              data['DEC'],
                              2. / 3600.,
                              maxmatch=1)
        data = data[m2]
        dist = dist[m1]
        distredux = path._redux_dr()
        if distredux.lower() == 'v302' or distredux.lower() == path._DR10REDUX:
            data = esutil.numpy_util.add_fields(data, [('DM05', float),
                                                       ('DM16', float),
                                                       ('DM50', float),
                                                       ('DM84', float),
                                                       ('DM95', float),
                                                       ('DMPEAK', float),
                                                       ('DMAVG', float),
                                                       ('SIG_DM', float),
                                                       ('DIST_SOL', float),
                                                       ('SIG_DISTSOL', float)])
            data['DM05'] = dist['DM05']
            data['DM16'] = dist['DM16']
            data['DM50'] = dist['DM50']
            data['DM84'] = dist['DM84']
            data['DM95'] = dist['DM95']
            data['DMPEAK'] = dist['DMPEAK']
            data['DMAVG'] = dist['DMAVG']
            data['SIG_DM'] = dist['SIG_DM']
            data['DIST_SOL'] = dist['DIST_SOL'] / 1000.
            data['SIG_DISTSOL'] = dist['SIG_DISTSOL'] / 1000.
        elif distredux.lower() == path._DR11REDUX:
            data = esutil.numpy_util.add_fields(data, [('DISO', float),
                                                       ('DMASS', float),
                                                       ('DISO_GAL', float),
                                                       ('DMASS_GAL', float)])
            data['DISO'] = dist['DISO'][:, 1]
            data['DMASS'] = dist['DMASS'][:, 1]
            data['DISO_GAL'] = dist['DISO_GAL'][:, 1]
            data['DMASS_GAL'] = dist['DMASS_GAL'][:, 1]
        elif distredux.lower() == path._DR12REDUX:
            data = esutil.numpy_util.add_fields(
                data, [('HIP_PLX', float), ('HIP_E_PLX', float),
                       ('RC_DIST', float), ('APOKASC_DIST_DIRECT', float),
                       ('BPG_DIST1_MEAN', float), ('HAYDEN_DIST_PEAK', float),
                       ('SCHULTHEIS_DIST', float)])
            data['HIP_PLX'] = dist['HIP_PLX']
            data['HIP_E_PLX'] = dist['HIP_E_PLX']
            data['RC_DIST'] = dist['RC_dist_pc']
            data[
                'APOKASC_DIST_DIRECT'] = dist['APOKASC_dist_direct_pc'] / 1000.
            data['BPG_DIST1_MEAN'] = dist['BPG_dist1_mean']
            data['HAYDEN_DIST_PEAK'] = 10.**(dist['HAYDEN_distmod_PEAK'] / 5. -
                                             2.)
            data['SCHULTHEIS_DIST'] = dist['SCHULTHEIS_dist']
    elif adddist:
        warnings.warn(
            "Distances not added because matching requires the uninstalled esutil module",
            RuntimeWarning)
    if _ESUTIL_LOADED and (path._APOGEE_REDUX.lower() == 'current' \
                               or 'l30' in path._APOGEE_REDUX.lower() \
                               or int(path._APOGEE_REDUX[1:]) > 600):
        data = esutil.numpy_util.add_fields(data, [('METALS', float),
                                                   ('ALPHAFE', float)])
        data['METALS'] = data['PARAM'][:, paramIndx('metals')]
        data['ALPHAFE'] = data['PARAM'][:, paramIndx('alpha')]
    return data
Exemple #2
0
def allStar(rmcommissioning=True,
            main=False,
            exclude_star_bad=False,
            exclude_star_warn=False,
            ak=True,
            akvers='targ',
            rmnovisits=False,
            adddist=False,
            distredux=None,
            rmdups=False,
            raw=False):
    """
    NAME:
       allStar
    PURPOSE:
       read the allStar file
    INPUT:
       rmcommissioning= (default: True) if True, only use data obtained after commissioning
       main= (default: False) if True, only select stars in the main survey
       exclude_star_bad= (False) if True, remove stars with the STAR_BAD flag set in ASPCAPFLAG
       exclude_star_warn= (False) if True, remove stars with the STAR_WARN flag set in ASPCAPFLAG
       ak= (default: True) only use objects for which dereddened mags exist
       akvers= 'targ' (default) or 'wise': use target AK (AK_TARG) or AK derived from all-sky WISE (AK_WISE)
       rmnovisits= (False) if True, remove stars with no good visits (to go into the combined spectrum); shouldn't be necessary
       adddist= (default: False) add distances (DR10/11 Hayden distances, DR12 combined distances)
       distredux= (default: DR default) reduction on which the distances are based
       rmdups= (False) if True, remove duplicates (very slow)
       raw= (False) if True, just return the raw file, read w/ fitsio
    OUTPUT:
       allStar data
    HISTORY:
       2013-09-06 - Written - Bovy (IAS)
    """
    filePath= path.allStarPath()
    if not os.path.exists(filePath):
        download.allStar()
    #read allStar file
    data= fitsio.read(path.allStarPath())
    if raw: return data
    #Remove duplicates, cache
    if rmdups:
        dupsFilename= path.allStarPath().replace('.fits','-nodups.fits')
        if os.path.exists(dupsFilename):
            data= fitsio.read(dupsFilename)
        else:
            sys.stdout.write('\r'+"Removing duplicates (might take a while) and caching the duplicate-free file ...\r")
            sys.stdout.flush()
            data= remove_duplicates(data)
            #Cache this file for subsequent use of rmdups
            fitsio.write(dupsFilename,data,clobber=True)
            sys.stdout.write('\r'+_ERASESTR+'\r')
            sys.stdout.flush()
    #Some cuts
    if rmcommissioning:
        indx= numpy.array(['apogee.n.c' in s for s in data['APSTAR_ID']])
        indx+= numpy.array(['apogee.s.c' in s for s in data['APSTAR_ID']])
        data= data[True-indx]
    if rmnovisits:
        indx= numpy.array([s.strip() != '' for s in data['VISITS']])
        data= data[indx]
    if main:
        indx= mainIndx(data)
        data= data[indx]
    if akvers.lower() == 'targ':
        aktag= 'AK_TARG'
    elif akvers.lower() == 'wise':
        aktag= 'AK_WISE'
    if ak:
        data= data[True-numpy.isnan(data[aktag])]
        data= data[(data[aktag] > -50.)]
    if exclude_star_bad:
        data= data[(data['ASPCAPFLAG'] & 2**23) == 0]
    if exclude_star_warn:
        data= data[(data['ASPCAPFLAG'] & 2**7) == 0]
    #Add dereddened J, H, and Ks
    aj= data[aktag]*2.5
    ah= data[aktag]*1.55
    data= esutil.numpy_util.add_fields(data,[('J0', float),
                                             ('H0', float),
                                             ('K0', float)])
    data['J0']= data['J']-aj
    data['H0']= data['H']-ah
    data['K0']= data['K']-data[aktag]
    data['J0'][(data[aktag] <= -50.)]= -9999.9999
    data['H0'][(data[aktag] <= -50.)]= -9999.9999
    data['K0'][(data[aktag] <= -50.)]= -9999.9999
    #Add distances
    if adddist:
        dist= fitsio.read(path.distPath(),1)
        h=esutil.htm.HTM()
        m1,m2,d12 = h.match(dist['RA'],dist['DEC'],
                             data['RA'],data['DEC'],
                             2./3600.,maxmatch=1)
        data= data[m2]
        dist= dist[m1]
        distredux= path._redux_dr()
        if distredux.lower() == 'v302' or distredux.lower() == path._DR10REDUX:
            data= esutil.numpy_util.add_fields(data,[('DM05', float),
                                                     ('DM16', float),
                                                     ('DM50', float),
                                                     ('DM84', float),
                                                     ('DM95', float),
                                                     ('DMPEAK', float),
                                                     ('DMAVG', float),
                                                     ('SIG_DM', float),
                                                     ('DIST_SOL', float),
                                                     ('SIG_DISTSOL', float)])
            data['DM05']= dist['DM05']
            data['DM16']= dist['DM16']
            data['DM50']= dist['DM50']
            data['DM84']= dist['DM84']
            data['DM95']= dist['DM95']
            data['DMPEAK']= dist['DMPEAK']
            data['DMAVG']= dist['DMAVG']
            data['SIG_DM']= dist['SIG_DM']
            data['DIST_SOL']= dist['DIST_SOL']/1000.
            data['SIG_DISTSOL']= dist['SIG_DISTSOL']/1000.
        elif distredux.lower() == path._DR11REDUX:
            data= esutil.numpy_util.add_fields(data,[('DISO', float),
                                                     ('DMASS', float),
                                                     ('DISO_GAL', float),
                                                     ('DMASS_GAL', float)])
            data['DISO']= dist['DISO'][:,1]
            data['DMASS']= dist['DMASS'][:,1]
            data['DISO_GAL']= dist['DISO_GAL'][:,1]
            data['DMASS_GAL']= dist['DMASS_GAL'][:,1]
        elif distredux.lower() == path._DR12REDUX:
            data= esutil.numpy_util.add_fields(data,[('HIP_PLX', float),
                                                     ('HIP_E_PLX', float),
                                                     ('RC_DIST', float),
                                                     ('APOKASC_DIST_DIRECT', float),
                                                     ('BPG_DIST1_MEAN', float),
                                                     ('HAYDEN_DIST_PEAK', float),
                                                     ('SCHULTHEIS_DIST', float)])
            data['HIP_PLX']= dist['HIP_PLX']
            data['HIP_E_PLX']= dist['HIP_E_PLX']
            data['RC_DIST']= dist['RC_dist_pc']
            data['APOKASC_DIST_DIRECT']= dist['APOKASC_dist_direct_pc']/1000.
            data['BPG_DIST1_MEAN']= dist['BPG_dist1_mean']
            data['HAYDEN_DIST_PEAK']= 10.**(dist['HAYDEN_distmod_PEAK']/5.-2.)
            data['SCHULTHEIS_DIST']= dist['SCHULTHEIS_dist']
    if path._APOGEE_REDUX.lower() == 'current' \
            or int(path._APOGEE_REDUX[1:]) > 600:
        data= esutil.numpy_util.add_fields(data,[('METALS', float),
                                                 ('ALPHAFE', float)])
        data['METALS']= data['PARAM'][:,paramIndx('metals')]
        data['ALPHAFE']= data['PARAM'][:,paramIndx('alpha')]
    return data
Exemple #3
0
def allStar(rmcommissioning=True,
            main=False,
            exclude_star_bad=False,
            exclude_star_warn=False,
            ak=True,
            akvers='targ',
            rmnovisits=False,
            use_astroNN=False,
            use_astroNN_abundances=False,
            use_astroNN_distances=False,
            use_astroNN_ages=False,
            adddist=False,
            distredux=None,
            rmdups=False,
            raw=False,
            mjd=58104,
            xmatch=None,
            **kwargs):
    """
    NAME:
       allStar
    PURPOSE:
       read the allStar file
    INPUT:
       rmcommissioning= (default: True) if True, only use data obtained after commissioning
       main= (default: False) if True, only select stars in the main survey
       exclude_star_bad= (False) if True, remove stars with the STAR_BAD flag set in ASPCAPFLAG
       exclude_star_warn= (False) if True, remove stars with the STAR_WARN flag set in ASPCAPFLAG
       ak= (default: True) only use objects for which dereddened mags exist
       akvers= 'targ' (default) or 'wise': use target AK (AK_TARG) or AK derived from all-sky WISE (AK_WISE)
       rmnovisits= (False) if True, remove stars with no good visits (to go into the combined spectrum); shouldn't be necessary
       use_astroNN= (False) if True, swap in astroNN (Leung & Bovy 2019a) parameters (get placed in, e.g., TEFF and TEFF_ERR), astroNN distances (Leung & Bovy 2019b), and astroNN ages (Mackereth, Bovy, Leung, et al. (2019)
       use_astroNN_abundances= (False) only swap in astroNN parameters and abundances, not distances and ages
       use_astroNN_distances= (False) only swap in astroNN distances, not  parameters and abundances and ages
       use_astroNN_ages= (False) only swap in astroNN ages, not  parameters and abundances and distances
       adddist= (default: False) add distances (DR10/11 Hayden distances, DR12 combined distances)
       distredux= (default: DR default) reduction on which the distances are based
       rmdups= (False) if True, remove duplicates (very slow)
       raw= (False) if True, just return the raw file, read w/ fitsio
       mjd= (58104) MJD of version for monthly internal pipeline runs
       xmatch= (None) uses gaia_tools.xmatch.cds to x-match to an external catalog (eg., Gaia DR2 for xmatch='vizier:I/345/gaia2') and caches the result for re-use; requires jobovy/gaia_tools
        +gaia_tools.xmatch.cds keywords 
    OUTPUT:
       allStar data[,xmatched table]
    HISTORY:
       2013-09-06 - Written - Bovy (IAS)
       2018-01-22 - Edited for new monthly pipeline runs - Bovy (UofT)
       2018-05-09 - Add xmatch - Bovy (UofT) 
       2018-10-20 - Add use_astroNN option - Bovy (UofT) 
       2018-02-15 - Add astroNN distances and corresponding options - Bovy (UofT) 
       2018-02-16 - Add astroNN ages and corresponding options - Bovy (UofT) 
    """
    filePath = path.allStarPath(mjd=mjd)
    if not os.path.exists(filePath):
        download.allStar(mjd=mjd)
    #read allStar file
    data = fitsread(path.allStarPath(mjd=mjd))
    #Add astroNN? astroNN file matched line-by-line to allStar, so match here
    # [ages file not matched line-by-line]
    if use_astroNN or kwargs.get('astroNN', False) or use_astroNN_abundances:
        _warn_astroNN_abundances()
        astroNNdata = astroNN()
        data = _swap_in_astroNN(data, astroNNdata)
    if use_astroNN or kwargs.get('astroNN', False) or use_astroNN_distances:
        _warn_astroNN_distances()
        astroNNdata = astroNNDistances()
        data = _add_astroNN_distances(data, astroNNdata)
    if use_astroNN or kwargs.get('astroNN', False) or use_astroNN_ages:
        _warn_astroNN_ages()
        astroNNdata = astroNNAges()
        data = _add_astroNN_ages(data, astroNNdata)
    if raw: return data
    #Remove duplicates, cache
    if rmdups:
        dupsFilename = path.allStarPath(mjd=mjd).replace(
            '.fits', '-nodups.fits')
        if os.path.exists(dupsFilename):
            data = fitsread(dupsFilename)
        else:
            sys.stdout.write(
                '\r' +
                "Removing duplicates (might take a while) and caching the duplicate-free file ...\r"
            )
            sys.stdout.flush()
            data = remove_duplicates(data)
            #Cache this file for subsequent use of rmdups
            fitswrite(dupsFilename, data, clobber=True)
            sys.stdout.write('\r' + _ERASESTR + '\r')
            sys.stdout.flush()
    if not xmatch is None:
        from gaia_tools.load import _xmatch_cds
        if rmdups:
            matchFilePath = dupsFilename
        else:
            matchFilePath = filePath
        if use_astroNN_ages:
            matchFilePath = matchFilePath.replace('rc-', 'rc-astroNN-ages-')
        ma, mai = _xmatch_cds(data, xmatch, filePath, **kwargs)
        data = data[mai]
    #Some cuts
    if rmcommissioning:
        try:
            indx = numpy.array(
                ['apogee.n.c'.encode('utf-8') in s for s in data['APSTAR_ID']])
            indx += numpy.array(
                ['apogee.s.c'.encode('utf-8') in s for s in data['APSTAR_ID']])
        except TypeError:
            indx = numpy.array(['apogee.n.c' in s for s in data['APSTAR_ID']])
            indx += numpy.array(['apogee.s.c' in s for s in data['APSTAR_ID']])
        data = data[True ^ indx]
        if not xmatch is None: ma = ma[True ^ indx]
    if rmnovisits:
        indx = numpy.array([s.strip() != '' for s in data['VISITS']])
        data = data[indx]
        if not xmatch is None: ma = ma[indx]
    if main:
        indx = mainIndx(data)
        data = data[indx]
        if not xmatch is None: ma = ma[indx]
    if akvers.lower() == 'targ':
        aktag = 'AK_TARG'
    elif akvers.lower() == 'wise':
        aktag = 'AK_WISE'
    if ak:
        if not xmatch is None: ma = ma[True ^ numpy.isnan(data[aktag])]
        data = data[True ^ numpy.isnan(data[aktag])]
        if not xmatch is None: ma = ma[(data[aktag] > -50.)]
        data = data[(data[aktag] > -50.)]
    if exclude_star_bad:
        if not xmatch is None: ma = ma[(data['ASPCAPFLAG'] & 2**23) == 0]
        data = data[(data['ASPCAPFLAG'] & 2**23) == 0]
    if exclude_star_warn:
        if not xmatch is None: ma = ma[(data['ASPCAPFLAG'] & 2**7) == 0]
        data = data[(data['ASPCAPFLAG'] & 2**7) == 0]
    #Add dereddened J, H, and Ks
    aj = data[aktag] * 2.5
    ah = data[aktag] * 1.55
    if _ESUTIL_LOADED:
        data = esutil.numpy_util.add_fields(data,
                                            [('J0', float), ('H0', float),
                                             ('K0', float)])
        data['J0'] = data['J'] - aj
        data['H0'] = data['H'] - ah
        data['K0'] = data['K'] - data[aktag]
        data['J0'][(data[aktag] <= -50.)] = -9999.9999
        data['H0'][(data[aktag] <= -50.)] = -9999.9999
        data['K0'][(data[aktag] <= -50.)] = -9999.9999
    else:
        warnings.warn(
            "Extinction-corrected J,H,K not added because esutil is not installed",
            RuntimeWarning)
    #Add distances
    if adddist and _ESUTIL_LOADED:
        dist = fitsread(path.distPath(), 1)
        h = esutil.htm.HTM()
        m1, m2, d12 = h.match(dist['RA'],
                              dist['DEC'],
                              data['RA'],
                              data['DEC'],
                              2. / 3600.,
                              maxmatch=1)
        data = data[m2]
        if not xmatch is None: ma = ma[m2]
        dist = dist[m1]
        distredux = path._redux_dr()
        if distredux.lower() == 'v302' or distredux.lower() == path._DR10REDUX:
            data = esutil.numpy_util.add_fields(data, [('DM05', float),
                                                       ('DM16', float),
                                                       ('DM50', float),
                                                       ('DM84', float),
                                                       ('DM95', float),
                                                       ('DMPEAK', float),
                                                       ('DMAVG', float),
                                                       ('SIG_DM', float),
                                                       ('DIST_SOL', float),
                                                       ('SIG_DISTSOL', float)])
            data['DM05'] = dist['DM05']
            data['DM16'] = dist['DM16']
            data['DM50'] = dist['DM50']
            data['DM84'] = dist['DM84']
            data['DM95'] = dist['DM95']
            data['DMPEAK'] = dist['DMPEAK']
            data['DMAVG'] = dist['DMAVG']
            data['SIG_DM'] = dist['SIG_DM']
            data['DIST_SOL'] = dist['DIST_SOL'] / 1000.
            data['SIG_DISTSOL'] = dist['SIG_DISTSOL'] / 1000.
        elif distredux.lower() == path._DR11REDUX:
            data = esutil.numpy_util.add_fields(data, [('DISO', float),
                                                       ('DMASS', float),
                                                       ('DISO_GAL', float),
                                                       ('DMASS_GAL', float)])
            data['DISO'] = dist['DISO'][:, 1]
            data['DMASS'] = dist['DMASS'][:, 1]
            data['DISO_GAL'] = dist['DISO_GAL'][:, 1]
            data['DMASS_GAL'] = dist['DMASS_GAL'][:, 1]
        elif distredux.lower() == path._DR12REDUX:
            data = esutil.numpy_util.add_fields(
                data, [('HIP_PLX', float), ('HIP_E_PLX', float),
                       ('RC_DIST', float), ('APOKASC_DIST_DIRECT', float),
                       ('BPG_DIST1_MEAN', float), ('HAYDEN_DIST_PEAK', float),
                       ('SCHULTHEIS_DIST', float)])
            data['HIP_PLX'] = dist['HIP_PLX']
            data['HIP_E_PLX'] = dist['HIP_E_PLX']
            data['RC_DIST'] = dist['RC_dist_pc']
            data[
                'APOKASC_DIST_DIRECT'] = dist['APOKASC_dist_direct_pc'] / 1000.
            data['BPG_DIST1_MEAN'] = dist['BPG_dist1_mean']
            data['HAYDEN_DIST_PEAK'] = 10.**(dist['HAYDEN_distmod_PEAK'] / 5. -
                                             2.)
            data['SCHULTHEIS_DIST'] = dist['SCHULTHEIS_dist']
    elif adddist:
        warnings.warn(
            "Distances not added because matching requires the uninstalled esutil module",
            RuntimeWarning)
    if _ESUTIL_LOADED and (path._APOGEE_REDUX.lower() == 'current' \
                               or 'l3' in path._APOGEE_REDUX.lower() \
                               or int(path._APOGEE_REDUX[1:]) > 600):
        data = esutil.numpy_util.add_fields(data, [('METALS', float),
                                                   ('ALPHAFE', float)])
        data['METALS'] = data['PARAM'][:, paramIndx('metals')]
        data['ALPHAFE'] = data['PARAM'][:, paramIndx('alpha')]
    if not xmatch is None:
        return (data, ma)
    else:
        return data
Exemple #4
0
def allStar(rmcommissioning=True,
            main=False,
            exclude_star_bad=False,
            exclude_star_warn=False,
            ak=True,
            akvers='targ',
            rmnovisits=False,
            use_astroNN=False,
            use_astroNN_abundances=False,
            use_astroNN_distances=False,          
            use_astroNN_ages=False,          
            adddist=False,
            distredux=None,
            rmdups=False,
            raw=False,
            mjd=58104,
            xmatch=None,**kwargs):
    """
    NAME:
       allStar
    PURPOSE:
       read the allStar file
    INPUT:
       rmcommissioning= (default: True) if True, only use data obtained after commissioning
       main= (default: False) if True, only select stars in the main survey
       exclude_star_bad= (False) if True, remove stars with the STAR_BAD flag set in ASPCAPFLAG
       exclude_star_warn= (False) if True, remove stars with the STAR_WARN flag set in ASPCAPFLAG
       ak= (default: True) only use objects for which dereddened mags exist
       akvers= 'targ' (default) or 'wise': use target AK (AK_TARG) or AK derived from all-sky WISE (AK_WISE)
       rmnovisits= (False) if True, remove stars with no good visits (to go into the combined spectrum); shouldn't be necessary
       use_astroNN= (False) if True, swap in astroNN (Leung & Bovy 2019a) parameters (get placed in, e.g., TEFF and TEFF_ERR), astroNN distances (Leung & Bovy 2019b), and astroNN ages (Mackereth, Bovy, Leung, et al. (2019)
       use_astroNN_abundances= (False) only swap in astroNN parameters and abundances, not distances and ages
       use_astroNN_distances= (False) only swap in astroNN distances, not  parameters and abundances and ages
       use_astroNN_ages= (False) only swap in astroNN ages, not  parameters and abundances and distances
       adddist= (default: False) add distances (DR10/11 Hayden distances, DR12 combined distances)
       distredux= (default: DR default) reduction on which the distances are based
       rmdups= (False) if True, remove duplicates (very slow)
       raw= (False) if True, just return the raw file, read w/ fitsio
       mjd= (58104) MJD of version for monthly internal pipeline runs
       xmatch= (None) uses gaia_tools.xmatch.cds to x-match to an external catalog (eg., Gaia DR2 for xmatch='vizier:I/345/gaia2') and caches the result for re-use; requires jobovy/gaia_tools
        +gaia_tools.xmatch.cds keywords 
    OUTPUT:
       allStar data[,xmatched table]
    HISTORY:
       2013-09-06 - Written - Bovy (IAS)
       2018-01-22 - Edited for new monthly pipeline runs - Bovy (UofT)
       2018-05-09 - Add xmatch - Bovy (UofT) 
       2018-10-20 - Add use_astroNN option - Bovy (UofT) 
       2018-02-15 - Add astroNN distances and corresponding options - Bovy (UofT) 
       2018-02-16 - Add astroNN ages and corresponding options - Bovy (UofT) 
    """
    filePath= path.allStarPath(mjd=mjd)
    if not os.path.exists(filePath):
        download.allStar(mjd=mjd)
    #read allStar file
    data= fitsread(path.allStarPath(mjd=mjd))
    #Add astroNN? astroNN file matched line-by-line to allStar, so match here
    # [ages file not matched line-by-line]
    if use_astroNN or kwargs.get('astroNN',False) or use_astroNN_abundances:
        _warn_astroNN_abundances()
        astroNNdata= astroNN()
        data= _swap_in_astroNN(data,astroNNdata)
    if use_astroNN or kwargs.get('astroNN',False) or use_astroNN_distances:
        _warn_astroNN_distances()
        astroNNdata= astroNNDistances()
        data= _add_astroNN_distances(data,astroNNdata)
    if use_astroNN or kwargs.get('astroNN',False) or use_astroNN_ages:
        _warn_astroNN_ages()
        astroNNdata= astroNNAges()
        data= _add_astroNN_ages(data,astroNNdata)
    if raw: return data
    #Remove duplicates, cache
    if rmdups:
        dupsFilename= path.allStarPath(mjd=mjd).replace('.fits','-nodups.fits')
        if os.path.exists(dupsFilename):
            data= fitsread(dupsFilename)
        else:
            sys.stdout.write('\r'+"Removing duplicates (might take a while) and caching the duplicate-free file ...\r")
            sys.stdout.flush()
            data= remove_duplicates(data)
            #Cache this file for subsequent use of rmdups
            fitswrite(dupsFilename,data,clobber=True)
            sys.stdout.write('\r'+_ERASESTR+'\r')
            sys.stdout.flush()
    if not xmatch is None:
        from gaia_tools.load import _xmatch_cds
        if rmdups:
            matchFilePath= dupsFilename
        else:
            matchFilePath= filePath
        if use_astroNN_ages:
            matchFilePath= matchFilePath.replace('rc-','rc-astroNN-ages-')
        ma,mai= _xmatch_cds(data,xmatch,filePath,**kwargs)
        data= data[mai]
    #Some cuts
    if rmcommissioning:
        try:
            indx= numpy.array(['apogee.n.c'.encode('utf-8') in s for s in data['APSTAR_ID']])
            indx+= numpy.array(['apogee.s.c'.encode('utf-8') in s for s in data['APSTAR_ID']])
        except TypeError:
            indx= numpy.array(['apogee.n.c' in s for s in data['APSTAR_ID']])
            indx+= numpy.array(['apogee.s.c' in s for s in data['APSTAR_ID']])
        data= data[True^indx]
        if not xmatch is None: ma= ma[True^indx]
    if rmnovisits:
        indx= numpy.array([s.strip() != '' for s in data['VISITS']])
        data= data[indx]
        if not xmatch is None: ma= ma[indx]
    if main:
        indx= mainIndx(data)
        data= data[indx]
        if not xmatch is None: ma= ma[indx]
    if akvers.lower() == 'targ':
        aktag= 'AK_TARG'
    elif akvers.lower() == 'wise':
        aktag= 'AK_WISE'
    if ak:
        if not xmatch is None: ma= ma[True^numpy.isnan(data[aktag])]
        data= data[True^numpy.isnan(data[aktag])]
        if not xmatch is None: ma= ma[(data[aktag] > -50.)]
        data= data[(data[aktag] > -50.)]
    if exclude_star_bad:
        if not xmatch is None: ma= ma[(data['ASPCAPFLAG'] & 2**23) == 0]
        data= data[(data['ASPCAPFLAG'] & 2**23) == 0]
    if exclude_star_warn:
        if not xmatch is None: ma= ma[(data['ASPCAPFLAG'] & 2**7) == 0]
        data= data[(data['ASPCAPFLAG'] & 2**7) == 0]
    #Add dereddened J, H, and Ks
    aj= data[aktag]*2.5
    ah= data[aktag]*1.55
    if _ESUTIL_LOADED:
        data= esutil.numpy_util.add_fields(data,[('J0', float),
                                                 ('H0', float),
                                                 ('K0', float)])
        data['J0']= data['J']-aj
        data['H0']= data['H']-ah
        data['K0']= data['K']-data[aktag]
        data['J0'][(data[aktag] <= -50.)]= -9999.9999
        data['H0'][(data[aktag] <= -50.)]= -9999.9999
        data['K0'][(data[aktag] <= -50.)]= -9999.9999
    else:
        warnings.warn("Extinction-corrected J,H,K not added because esutil is not installed",RuntimeWarning)
    #Add distances
    if adddist and _ESUTIL_LOADED:
        dist= fitsread(path.distPath(),1)
        h=esutil.htm.HTM()
        m1,m2,d12 = h.match(dist['RA'],dist['DEC'],
                             data['RA'],data['DEC'],
                             2./3600.,maxmatch=1)
        data= data[m2]
        if not xmatch is None: ma= ma[m2]
        dist= dist[m1]
        distredux= path._redux_dr()
        if distredux.lower() == 'v302' or distredux.lower() == path._DR10REDUX:
            data= esutil.numpy_util.add_fields(data,[('DM05', float),
                                                     ('DM16', float),
                                                     ('DM50', float),
                                                     ('DM84', float),
                                                     ('DM95', float),
                                                     ('DMPEAK', float),
                                                     ('DMAVG', float),
                                                     ('SIG_DM', float),
                                                     ('DIST_SOL', float),
                                                     ('SIG_DISTSOL', float)])
            data['DM05']= dist['DM05']
            data['DM16']= dist['DM16']
            data['DM50']= dist['DM50']
            data['DM84']= dist['DM84']
            data['DM95']= dist['DM95']
            data['DMPEAK']= dist['DMPEAK']
            data['DMAVG']= dist['DMAVG']
            data['SIG_DM']= dist['SIG_DM']
            data['DIST_SOL']= dist['DIST_SOL']/1000.
            data['SIG_DISTSOL']= dist['SIG_DISTSOL']/1000.
        elif distredux.lower() == path._DR11REDUX:
            data= esutil.numpy_util.add_fields(data,[('DISO', float),
                                                     ('DMASS', float),
                                                     ('DISO_GAL', float),
                                                     ('DMASS_GAL', float)])
            data['DISO']= dist['DISO'][:,1]
            data['DMASS']= dist['DMASS'][:,1]
            data['DISO_GAL']= dist['DISO_GAL'][:,1]
            data['DMASS_GAL']= dist['DMASS_GAL'][:,1]
        elif distredux.lower() == path._DR12REDUX:
            data= esutil.numpy_util.add_fields(data,[('HIP_PLX', float),
                                                     ('HIP_E_PLX', float),
                                                     ('RC_DIST', float),
                                                     ('APOKASC_DIST_DIRECT', float),
                                                     ('BPG_DIST1_MEAN', float),
                                                     ('HAYDEN_DIST_PEAK', float),
                                                     ('SCHULTHEIS_DIST', float)])
            data['HIP_PLX']= dist['HIP_PLX']
            data['HIP_E_PLX']= dist['HIP_E_PLX']
            data['RC_DIST']= dist['RC_dist_pc']
            data['APOKASC_DIST_DIRECT']= dist['APOKASC_dist_direct_pc']/1000.
            data['BPG_DIST1_MEAN']= dist['BPG_dist1_mean']
            data['HAYDEN_DIST_PEAK']= 10.**(dist['HAYDEN_distmod_PEAK']/5.-2.)
            data['SCHULTHEIS_DIST']= dist['SCHULTHEIS_dist']
    elif adddist:
        warnings.warn("Distances not added because matching requires the uninstalled esutil module",RuntimeWarning)
    if _ESUTIL_LOADED and (path._APOGEE_REDUX.lower() == 'current' \
                               or 'l3' in path._APOGEE_REDUX.lower() \
                               or int(path._APOGEE_REDUX[1:]) > 600):
        data= esutil.numpy_util.add_fields(data,[('METALS', float),
                                                 ('ALPHAFE', float)])
        data['METALS']= data['PARAM'][:,paramIndx('metals')]
        data['ALPHAFE']= data['PARAM'][:,paramIndx('alpha')]
    if not xmatch is None:
        return (data,ma)
    else:
        return data