Ejemplo n.º 1
0
def test_sdss():
    """ Test SDSS DR7 -- This is very slow..
    """
    # All
    sdss_dr7_all = LLSSurvey.load_SDSS_DR7(sample='all')
    assert sdss_dr7_all.nsys == 1935
    # Stat
    sdss_dr7_stat = LLSSurvey.load_SDSS_DR7()
    assert len(sdss_dr7_stat.NHI) == 254
Ejemplo n.º 2
0
def test_sdss():
    """ Test SDSS DR7 -- This is very slow..
    """
    # All
    sdss_dr7_all = LLSSurvey.load_SDSS_DR7(sample='all')
    assert sdss_dr7_all.nsys == 1935
    # Stat
    sdss_dr7_stat = LLSSurvey.load_SDSS_DR7()
    assert len(sdss_dr7_stat.NHI) == 254
Ejemplo n.º 3
0
def test_z3mage():
    """ Test z~3 MagE
    """
    # All
    z3mage = LLSSurvey.load_mage_z3()
    assert z3mage.nsys == 60
    assert len(z3mage.sightlines) == 105
    # Non-Color
    z3mage_NC = LLSSurvey.load_mage_z3(sample='non-color')
    assert z3mage_NC.nsys == 32
    assert len(z3mage_NC.sightlines) == 61
Ejemplo n.º 4
0
def test_z3mage():
    """ Test z~3 MagE
    """
    # All
    z3mage = LLSSurvey.load_mage_z3()
    assert z3mage.nsys == 60
    assert len(z3mage.sightlines) == 105
    # Non-Color
    z3mage_NC = LLSSurvey.load_mage_z3(sample='non-color')
    assert z3mage_NC.nsys == 32
    assert len(z3mage_NC.sightlines) == 61
Ejemplo n.º 5
0
def test_sdss():
    """ Test SDSS DR7 -- This is very slow..
    """
    # All
    sdss_dr7_all = LLSSurvey.load_SDSS_DR7(sample='all')
    assert sdss_dr7_all.nsys == 1935
    # Stat
    sdss_dr7_stat = LLSSurvey.load_SDSS_DR7()
    assert len(sdss_dr7_stat.NHI) == 191
    #
    z_bins = np.array([3.5, 3.65, 3.9, 4.1, 4.4])
    lz, sig_lz_low, sig_lz_up = sdss_dr7_stat.binned_loz(z_bins,
                                                         NHI_mnx=(17.49, 23.))
Ejemplo n.º 6
0
def test_hst():
    """ Test HST surveys
    """
    # ACS
    acs = LLSSurvey.load_HST_ACS()
    assert acs.nsys == 34
    assert len(acs.sightlines) == 18
    # WFC3
    wfc3 = LLSSurvey.load_HST_WFC3()
    assert wfc3.nsys == 91
    assert len(wfc3.sightlines) == 53
    # Combined
    HST_LLS = wfc3 + acs
    assert HST_LLS.nsys == 125
    assert len(HST_LLS.sightlines) == 71
Ejemplo n.º 7
0
def test_hst():
    """ Test HST surveys
    """
    # ACS
    acs = LLSSurvey.load_HST_ACS()
    assert acs.nsys == 34
    assert len(acs.sightlines) == 18
    # WFC3
    wfc3 = LLSSurvey.load_HST_WFC3()
    assert wfc3.nsys == 91
    assert len(wfc3.sightlines) == 53
    # Combined
    HST_LLS = wfc3 + acs
    assert HST_LLS.nsys == 125
    assert len(HST_LLS.sightlines) == 71
Ejemplo n.º 8
0
def test_z3mage():
    """ Test z~3 MagE
    """
    # All
    z3mage = LLSSurvey.load_mage_z3()
    assert z3mage.nsys == 60
    assert len(z3mage.sightlines) == 105
    # Non-Color
    z3mage_NC = LLSSurvey.load_mage_z3(sample='non-color')
    assert z3mage_NC.nsys == 27
    assert len(z3mage_NC.sightlines) == 61
    # Stats
    lz, sig_lz_low, sig_lz_up = z3mage_NC.binned_loz([2.6, 3.],
                                                     NHI_mnx=(17.49, 23.))
    assert np.isclose(lz[0], 1.20521669)
Ejemplo n.º 9
0
def test_read_hdlls_dr1():   # This might actually be local now..
    hdlls = LLSSurvey.load_HDLLS()
    assert hdlls.nsys == 157

    CII_clms = hdlls.ions((6,2))
    gdCII = np.where(CII_clms['flag_N']>0)[0]
    assert len(gdCII) == 103
Ejemplo n.º 10
0
def test_read_hdlls_dr1():
    hdlls = LLSSurvey.load_HDLLS()
    assert hdlls.nsys == 157

    CII_clms = hdlls.ions((6,2))
    gdCII = np.where(CII_clms['flag_N']>0)[0]
    assert len(gdCII) == 103
Ejemplo n.º 11
0
def test_read_hdlls_dr1():  # This might actually be local now..
    hdlls = LLSSurvey.load_HDLLS()
    assert hdlls.nsys == 157

    CII_clms = hdlls.ions((6, 2))
    gdCII = np.where(CII_clms['flag_N'] > 0)[0]
    assert len(gdCII) == 103
Ejemplo n.º 12
0
def test_hst():
    """ Test HST surveys
    """
    # ACS
    acs = LLSSurvey.load_HST_ACS()
    assert acs.nsys == 9
    assert len(acs.sightlines) == 18
    # WFC3
    wfc3 = LLSSurvey.load_HST_WFC3()
    #assert wfc3.nsys == 91
    assert wfc3.nsys == 32  # Was 30.  I think the NHI = 17.49 mattered
    assert len(wfc3.sightlines) == 53
    # Combined
    HST_LLS = wfc3 + acs
    #assert HST_LLS.nsys == 125
    assert HST_LLS.nsys == 41  # Was 39
    assert len(HST_LLS.sightlines) == 71
Ejemplo n.º 13
0
def test_load_ribaudo11():
    ribaudo11 = LLSSurvey.load_ribaudo()
    z, gz = ribaudo13.calculate_gz()
    assert gz[0] == 1
    assert gz[-1] == 3
    assert ribaudo11.nsys == 50
    # Stats
    lz, sig_lz_low, sig_lz_up = ribaudo11.binned_loz(
        [0.242, 1.078, 1.544, 1.947], NHI_mnx=(17.49, 23.))
Ejemplo n.º 14
0
def test_dat_list():
    """JXP format :: Likely to be Deprecated
    """
    # LLS Survey
    if os.getenv('LLSTREE') is None:
        assert True
        return
    # Load
    lls = LLSSurvey.from_flist('Lists/lls_metals.lst', tree=os.getenv('LLSTREE'))
    # tests
    np.testing.assert_allclose(lls.NHI[0], 19.25)
    assert lls.nsys == 165
Ejemplo n.º 15
0
def test_dat_list():
    """JXP format :: Likely to be Deprecated
    """
    # LLS Survey
    if os.getenv('LLSTREE') is None:
        assert True
        return
    # Load
    lls = LLSSurvey.from_flist('Lists/lls_metals.lst',
                               tree=os.getenv('LLSTREE'))
    # tests
    np.testing.assert_allclose(lls.NHI[0], 19.25)
    assert lls.nsys == 165
Ejemplo n.º 16
0
def test_gz():
    # All
    z3mage = LLSSurvey.load_mage_z3()
    zeval, gz = z3mage.calculate_gz()
    assert gz[4000] == 67
    np.testing.assert_allclose(zeval[4000], 2.8705998897560931)
Ejemplo n.º 17
0
def json_to_sdss_dlasurvey(json_file, sdss_survey, add_pf=True, debug=False):
    """ Convert JSON output file to a DLASurvey object
    Assumes SDSS bookkeeping for sightlines (i.e. PLATE, FIBER)

    Parameters
    ----------
    json_file : str
      Full path to the JSON results file
    sdss_survey : DLASurvey
      SDSS survey, usually human (e.g. JXP for DR5)
    add_pf : bool, optional
      Add plate/fiber to DLAs in sdss_survey

    Returns
    -------
    ml_survey : LLSSurvey
      Survey object for the LLS

    """
    print("Loading SDSS Survey from JSON file {:s}".format(json_file))
    # imports
    from pyigm.abssys.dla import DLASystem
    from pyigm.abssys.lls import LLSSystem
    # Fiber key
    for fkey in ['FIBER', 'FIBER_ID', 'FIB']:
        if fkey in sdss_survey.sightlines.keys():
            break
    # Read
    ml_results = ltu.loadjson(json_file)
    use_platef = False
    if 'plate' in ml_results[0].keys():
        use_platef = True
    else:
        if 'id' in ml_results[0].keys():
            use_id = True
    # Init
    #idict = dict(plate=[], fiber=[], classification_confidence=[],  # FOR v2
    #             classification=[], ra=[], dec=[])
    idict = dict(ra=[], dec=[])
    if use_platef:
        for key in ['plate', 'fiber', 'mjd']:
            idict[key] = []
    ml_tbl = Table()
    ml_survey = LLSSurvey()
    systems = []
    in_ml = np.array([False]*len(sdss_survey.sightlines))
    # Loop
    for obj in ml_results:
        # Sightline
        for key in idict.keys():
            idict[key].append(obj[key])
        # DLAs
        #if debug:
        #    if (obj['plate'] == 1366) & (obj['fiber'] == 614):
        #        sv_coord = SkyCoord(ra=obj['ra'], dec=obj['dec'], unit='deg')
        #        print("GOT A MATCH IN RESULTS FILE")
        for idla in obj['dlas']:
            """
            dla = DLASystem((sdss_survey.sightlines['RA'][mt[0]],
                             sdss_survey.sightlines['DEC'][mt[0]]),
                            idla['spectrum']/(1215.6701)-1., None,
                            idla['column_density'])
            """
            if idla['z_dla'] < 1.8:
                continue
            isys = LLSSystem((obj['ra'],obj['dec']),
                    idla['z_dla'], None, NHI=idla['column_density'], zem=obj['z_qso'])
            isys.confidence = idla['dla_confidence']
            if use_platef:
                isys.plate = obj['plate']
                isys.fiber = obj['fiber']
            elif use_id:
                plate, fiber = [int(spl) for spl in obj['id'].split('-')]
                isys.plate = plate
                isys.fiber = fiber
            # Save
            systems.append(isys)
    # Connect to sightlines
    ml_coord = SkyCoord(ra=idict['ra'], dec=idict['dec'], unit='deg')
    s_coord = SkyCoord(ra=sdss_survey.sightlines['RA'], dec=sdss_survey.sightlines['DEC'], unit='deg')
    idx, d2d, d3d = match_coordinates_sky(s_coord, ml_coord, nthneighbor=1)
    used = d2d < 1.*u.arcsec
    for iidx in np.where(~used)[0]:
        print("Sightline RA={:g}, DEC={:g} was not used".format(sdss_survey.sightlines['RA'][iidx],
                                                                sdss_survey.sightlines['DEC'][iidx]))
    # Add plate/fiber to statistical DLAs
    if add_pf:
        dla_coord = sdss_survey.coord
        idx2, d2d, d3d = match_coordinates_sky(dla_coord, s_coord, nthneighbor=1)
        if np.min(d2d.to('arcsec').value) > 1.:
            raise ValueError("Bad match to sightlines")
        for jj,igd in enumerate(np.where(sdss_survey.mask)[0]):
            dla = sdss_survey._abs_sys[igd]
            try:
                dla.plate = sdss_survey.sightlines['PLATE'][idx2[jj]]
            except IndexError:
                pdb.set_trace()
            dla.fiber = sdss_survey.sightlines[fkey][idx2[jj]]
    # Finish
    ml_survey._abs_sys = systems
    if debug:
        ml2_coord = ml_survey.coord
        minsep = np.min(sv_coord.separation(ml2_coord))
        minsep2 = np.min(sv_coord.separation(s_coord))
        tmp = sdss_survey.sightlines[used]
        t_coord = SkyCoord(ra=tmp['RA'], dec=tmp['DEC'], unit='deg')
        minsep3 = np.min(sv_coord.separation(t_coord))
        pdb.set_trace()
    ml_survey.sightlines = sdss_survey.sightlines[used]
    for key in idict.keys():
        ml_tbl[key] = idict[key]
    ml_survey.ml_tbl = ml_tbl
    # Return
    return ml_survey
Ejemplo n.º 18
0
def test_gz():
    # All
    z3mage = LLSSurvey.load_mage_z3()
    zeval, gz = z3mage.calculate_gz()
    assert gz[4000] == 67
    np.testing.assert_allclose(zeval[4000], 2.8705998897560931)
Ejemplo n.º 19
0
def test_read_hdlls_dr1_simple():
    hdlls = LLSSurvey.load_HDLLS(load_sys=False)
    assert hdlls.nsys == 157
Ejemplo n.º 20
0
def load_ml_file(pred_file):
    """ Load the search results from the CNN into a DLASurvey object
    Parameters
    ----------
    pred_file

    Returns
    -------
    ml_llssurvey: LLSSurvey
    ml_dlasusrvey: DLASurvey
    """
    print("Loading {:s}.  Please be patient..".format(pred_file))
    # Read
    ml_results = ltu.loadjson(pred_file)
    use_platef = False
    if 'plate' in ml_results[0].keys():
        use_platef = True
    else:
        if 'id' in ml_results[0].keys():
            use_id = True
    # Init
    idict = dict(ra=[], dec=[], plate=[], fiber=[])
    if use_platef:
        for key in ['plate', 'fiber', 'mjd']:
            idict[key] = []
    dlasystems = []
    llssystems = []

    # Generate coords to speed things up
    for obj in ml_results:
        for key in ['ra', 'dec']:
            idict[key].append(obj[key])
    ml_coords = SkyCoord(ra=idict['ra'], dec=idict['dec'], unit='deg')
    ra_names = ml_coords.icrs.ra.to_string(unit=u.hour, sep='', pad=True)
    dec_names = ml_coords.icrs.dec.to_string(sep='', pad=True, alwayssign=True)
    vlim = [-500., 500.] * u.km / u.s
    dcoord = SkyCoord(ra=0., dec=0., unit='deg')

    # Loop on list
    didx, lidx = [], []
    print("Looping on sightlines..")
    for tt, obj in enumerate(ml_results):
        #if (tt % 100) == 0:
        #    print('tt: {:d}'.format(tt))
        # Sightline
        if use_id:
            plate, fiber = [int(spl) for spl in obj['id'].split('-')]
            idict['plate'].append(plate)
            idict['fiber'].append(fiber)

        # Systems
        for ss, syskey in enumerate(['dlas', 'subdlas']):
            for idla in obj[syskey]:
                name = 'J{:s}{:s}_z{:.3f}'.format(ra_names[tt], dec_names[tt],
                                                  idla['z_dla'])
                if ss == 0:
                    isys = DLASystem(dcoord,
                                     idla['z_dla'],
                                     vlim,
                                     NHI=idla['column_density'],
                                     zem=obj['z_qso'],
                                     name=name)
                else:
                    isys = LLSSystem(dcoord,
                                     idla['z_dla'],
                                     vlim,
                                     NHI=idla['column_density'],
                                     zem=obj['z_qso'],
                                     name=name)
                isys.confidence = idla['dla_confidence']
                isys.s2n = idla['s2n']
                if use_platef:
                    isys.plate = obj['plate']
                    isys.fiber = obj['fiber']
                elif use_id:
                    isys.plate = plate
                    isys.fiber = fiber
                # Save
                if ss == 0:
                    didx.append(tt)
                    dlasystems.append(isys)
                else:
                    lidx.append(tt)
                    llssystems.append(isys)
    # Generate sightline tables
    sightlines = Table()
    sightlines['RA'] = idict['ra']
    sightlines['DEC'] = idict['dec']
    sightlines['PLATE'] = idict['plate']
    sightlines['FIBERID'] = idict['fiber']
    # Surveys
    ml_llssurvey = LLSSurvey()
    ml_llssurvey.sightlines = sightlines.copy()
    ml_llssurvey._abs_sys = llssystems
    ml_llssurvey.coords = ml_coords[np.array(lidx)]

    ml_dlasurvey = DLASurvey()
    ml_dlasurvey.sightlines = sightlines.copy()
    ml_dlasurvey._abs_sys = dlasystems
    ml_dlasurvey.coords = ml_coords[np.array(didx)]

    # Return
    return ml_llssurvey, ml_dlasurvey
Ejemplo n.º 21
0
def test_read_hdlls_dr1_simple():
    hdlls = LLSSurvey.load_HDLLS(load_sys=False)
    assert hdlls.nsys == 157