def test_sdss(): """ Test SDSS DR7 -- This is very slow.. """ # All sdss_dr7_all = LLSSurvey.load_SDSS_DR7(sample='all') assert sdss_dr7_all.nsys == 1935 # Stat sdss_dr7_stat = LLSSurvey.load_SDSS_DR7() assert len(sdss_dr7_stat.NHI) == 254
def test_z3mage(): """ Test z~3 MagE """ # All z3mage = LLSSurvey.load_mage_z3() assert z3mage.nsys == 60 assert len(z3mage.sightlines) == 105 # Non-Color z3mage_NC = LLSSurvey.load_mage_z3(sample='non-color') assert z3mage_NC.nsys == 32 assert len(z3mage_NC.sightlines) == 61
def test_sdss(): """ Test SDSS DR7 -- This is very slow.. """ # All sdss_dr7_all = LLSSurvey.load_SDSS_DR7(sample='all') assert sdss_dr7_all.nsys == 1935 # Stat sdss_dr7_stat = LLSSurvey.load_SDSS_DR7() assert len(sdss_dr7_stat.NHI) == 191 # z_bins = np.array([3.5, 3.65, 3.9, 4.1, 4.4]) lz, sig_lz_low, sig_lz_up = sdss_dr7_stat.binned_loz(z_bins, NHI_mnx=(17.49, 23.))
def test_hst(): """ Test HST surveys """ # ACS acs = LLSSurvey.load_HST_ACS() assert acs.nsys == 34 assert len(acs.sightlines) == 18 # WFC3 wfc3 = LLSSurvey.load_HST_WFC3() assert wfc3.nsys == 91 assert len(wfc3.sightlines) == 53 # Combined HST_LLS = wfc3 + acs assert HST_LLS.nsys == 125 assert len(HST_LLS.sightlines) == 71
def test_z3mage(): """ Test z~3 MagE """ # All z3mage = LLSSurvey.load_mage_z3() assert z3mage.nsys == 60 assert len(z3mage.sightlines) == 105 # Non-Color z3mage_NC = LLSSurvey.load_mage_z3(sample='non-color') assert z3mage_NC.nsys == 27 assert len(z3mage_NC.sightlines) == 61 # Stats lz, sig_lz_low, sig_lz_up = z3mage_NC.binned_loz([2.6, 3.], NHI_mnx=(17.49, 23.)) assert np.isclose(lz[0], 1.20521669)
def test_read_hdlls_dr1(): # This might actually be local now.. hdlls = LLSSurvey.load_HDLLS() assert hdlls.nsys == 157 CII_clms = hdlls.ions((6,2)) gdCII = np.where(CII_clms['flag_N']>0)[0] assert len(gdCII) == 103
def test_read_hdlls_dr1(): hdlls = LLSSurvey.load_HDLLS() assert hdlls.nsys == 157 CII_clms = hdlls.ions((6,2)) gdCII = np.where(CII_clms['flag_N']>0)[0] assert len(gdCII) == 103
def test_read_hdlls_dr1(): # This might actually be local now.. hdlls = LLSSurvey.load_HDLLS() assert hdlls.nsys == 157 CII_clms = hdlls.ions((6, 2)) gdCII = np.where(CII_clms['flag_N'] > 0)[0] assert len(gdCII) == 103
def test_hst(): """ Test HST surveys """ # ACS acs = LLSSurvey.load_HST_ACS() assert acs.nsys == 9 assert len(acs.sightlines) == 18 # WFC3 wfc3 = LLSSurvey.load_HST_WFC3() #assert wfc3.nsys == 91 assert wfc3.nsys == 32 # Was 30. I think the NHI = 17.49 mattered assert len(wfc3.sightlines) == 53 # Combined HST_LLS = wfc3 + acs #assert HST_LLS.nsys == 125 assert HST_LLS.nsys == 41 # Was 39 assert len(HST_LLS.sightlines) == 71
def test_load_ribaudo11(): ribaudo11 = LLSSurvey.load_ribaudo() z, gz = ribaudo13.calculate_gz() assert gz[0] == 1 assert gz[-1] == 3 assert ribaudo11.nsys == 50 # Stats lz, sig_lz_low, sig_lz_up = ribaudo11.binned_loz( [0.242, 1.078, 1.544, 1.947], NHI_mnx=(17.49, 23.))
def test_dat_list(): """JXP format :: Likely to be Deprecated """ # LLS Survey if os.getenv('LLSTREE') is None: assert True return # Load lls = LLSSurvey.from_flist('Lists/lls_metals.lst', tree=os.getenv('LLSTREE')) # tests np.testing.assert_allclose(lls.NHI[0], 19.25) assert lls.nsys == 165
def test_gz(): # All z3mage = LLSSurvey.load_mage_z3() zeval, gz = z3mage.calculate_gz() assert gz[4000] == 67 np.testing.assert_allclose(zeval[4000], 2.8705998897560931)
def json_to_sdss_dlasurvey(json_file, sdss_survey, add_pf=True, debug=False): """ Convert JSON output file to a DLASurvey object Assumes SDSS bookkeeping for sightlines (i.e. PLATE, FIBER) Parameters ---------- json_file : str Full path to the JSON results file sdss_survey : DLASurvey SDSS survey, usually human (e.g. JXP for DR5) add_pf : bool, optional Add plate/fiber to DLAs in sdss_survey Returns ------- ml_survey : LLSSurvey Survey object for the LLS """ print("Loading SDSS Survey from JSON file {:s}".format(json_file)) # imports from pyigm.abssys.dla import DLASystem from pyigm.abssys.lls import LLSSystem # Fiber key for fkey in ['FIBER', 'FIBER_ID', 'FIB']: if fkey in sdss_survey.sightlines.keys(): break # Read ml_results = ltu.loadjson(json_file) use_platef = False if 'plate' in ml_results[0].keys(): use_platef = True else: if 'id' in ml_results[0].keys(): use_id = True # Init #idict = dict(plate=[], fiber=[], classification_confidence=[], # FOR v2 # classification=[], ra=[], dec=[]) idict = dict(ra=[], dec=[]) if use_platef: for key in ['plate', 'fiber', 'mjd']: idict[key] = [] ml_tbl = Table() ml_survey = LLSSurvey() systems = [] in_ml = np.array([False]*len(sdss_survey.sightlines)) # Loop for obj in ml_results: # Sightline for key in idict.keys(): idict[key].append(obj[key]) # DLAs #if debug: # if (obj['plate'] == 1366) & (obj['fiber'] == 614): # sv_coord = SkyCoord(ra=obj['ra'], dec=obj['dec'], unit='deg') # print("GOT A MATCH IN RESULTS FILE") for idla in obj['dlas']: """ dla = DLASystem((sdss_survey.sightlines['RA'][mt[0]], sdss_survey.sightlines['DEC'][mt[0]]), idla['spectrum']/(1215.6701)-1., None, idla['column_density']) """ if idla['z_dla'] < 1.8: continue isys = LLSSystem((obj['ra'],obj['dec']), idla['z_dla'], None, NHI=idla['column_density'], zem=obj['z_qso']) isys.confidence = idla['dla_confidence'] if use_platef: isys.plate = obj['plate'] isys.fiber = obj['fiber'] elif use_id: plate, fiber = [int(spl) for spl in obj['id'].split('-')] isys.plate = plate isys.fiber = fiber # Save systems.append(isys) # Connect to sightlines ml_coord = SkyCoord(ra=idict['ra'], dec=idict['dec'], unit='deg') s_coord = SkyCoord(ra=sdss_survey.sightlines['RA'], dec=sdss_survey.sightlines['DEC'], unit='deg') idx, d2d, d3d = match_coordinates_sky(s_coord, ml_coord, nthneighbor=1) used = d2d < 1.*u.arcsec for iidx in np.where(~used)[0]: print("Sightline RA={:g}, DEC={:g} was not used".format(sdss_survey.sightlines['RA'][iidx], sdss_survey.sightlines['DEC'][iidx])) # Add plate/fiber to statistical DLAs if add_pf: dla_coord = sdss_survey.coord idx2, d2d, d3d = match_coordinates_sky(dla_coord, s_coord, nthneighbor=1) if np.min(d2d.to('arcsec').value) > 1.: raise ValueError("Bad match to sightlines") for jj,igd in enumerate(np.where(sdss_survey.mask)[0]): dla = sdss_survey._abs_sys[igd] try: dla.plate = sdss_survey.sightlines['PLATE'][idx2[jj]] except IndexError: pdb.set_trace() dla.fiber = sdss_survey.sightlines[fkey][idx2[jj]] # Finish ml_survey._abs_sys = systems if debug: ml2_coord = ml_survey.coord minsep = np.min(sv_coord.separation(ml2_coord)) minsep2 = np.min(sv_coord.separation(s_coord)) tmp = sdss_survey.sightlines[used] t_coord = SkyCoord(ra=tmp['RA'], dec=tmp['DEC'], unit='deg') minsep3 = np.min(sv_coord.separation(t_coord)) pdb.set_trace() ml_survey.sightlines = sdss_survey.sightlines[used] for key in idict.keys(): ml_tbl[key] = idict[key] ml_survey.ml_tbl = ml_tbl # Return return ml_survey
def test_read_hdlls_dr1_simple(): hdlls = LLSSurvey.load_HDLLS(load_sys=False) assert hdlls.nsys == 157
def load_ml_file(pred_file): """ Load the search results from the CNN into a DLASurvey object Parameters ---------- pred_file Returns ------- ml_llssurvey: LLSSurvey ml_dlasusrvey: DLASurvey """ print("Loading {:s}. Please be patient..".format(pred_file)) # Read ml_results = ltu.loadjson(pred_file) use_platef = False if 'plate' in ml_results[0].keys(): use_platef = True else: if 'id' in ml_results[0].keys(): use_id = True # Init idict = dict(ra=[], dec=[], plate=[], fiber=[]) if use_platef: for key in ['plate', 'fiber', 'mjd']: idict[key] = [] dlasystems = [] llssystems = [] # Generate coords to speed things up for obj in ml_results: for key in ['ra', 'dec']: idict[key].append(obj[key]) ml_coords = SkyCoord(ra=idict['ra'], dec=idict['dec'], unit='deg') ra_names = ml_coords.icrs.ra.to_string(unit=u.hour, sep='', pad=True) dec_names = ml_coords.icrs.dec.to_string(sep='', pad=True, alwayssign=True) vlim = [-500., 500.] * u.km / u.s dcoord = SkyCoord(ra=0., dec=0., unit='deg') # Loop on list didx, lidx = [], [] print("Looping on sightlines..") for tt, obj in enumerate(ml_results): #if (tt % 100) == 0: # print('tt: {:d}'.format(tt)) # Sightline if use_id: plate, fiber = [int(spl) for spl in obj['id'].split('-')] idict['plate'].append(plate) idict['fiber'].append(fiber) # Systems for ss, syskey in enumerate(['dlas', 'subdlas']): for idla in obj[syskey]: name = 'J{:s}{:s}_z{:.3f}'.format(ra_names[tt], dec_names[tt], idla['z_dla']) if ss == 0: isys = DLASystem(dcoord, idla['z_dla'], vlim, NHI=idla['column_density'], zem=obj['z_qso'], name=name) else: isys = LLSSystem(dcoord, idla['z_dla'], vlim, NHI=idla['column_density'], zem=obj['z_qso'], name=name) isys.confidence = idla['dla_confidence'] isys.s2n = idla['s2n'] if use_platef: isys.plate = obj['plate'] isys.fiber = obj['fiber'] elif use_id: isys.plate = plate isys.fiber = fiber # Save if ss == 0: didx.append(tt) dlasystems.append(isys) else: lidx.append(tt) llssystems.append(isys) # Generate sightline tables sightlines = Table() sightlines['RA'] = idict['ra'] sightlines['DEC'] = idict['dec'] sightlines['PLATE'] = idict['plate'] sightlines['FIBERID'] = idict['fiber'] # Surveys ml_llssurvey = LLSSurvey() ml_llssurvey.sightlines = sightlines.copy() ml_llssurvey._abs_sys = llssystems ml_llssurvey.coords = ml_coords[np.array(lidx)] ml_dlasurvey = DLASurvey() ml_dlasurvey.sightlines = sightlines.copy() ml_dlasurvey._abs_sys = dlasystems ml_dlasurvey.coords = ml_coords[np.array(didx)] # Return return ml_llssurvey, ml_dlasurvey