def test_sdss():
""" Test SDSS DR7 -- This is very slow..
"""
# All
sdss_dr7_all = LLSSurvey.load_SDSS_DR7(sample='all')
assert sdss_dr7_all.nsys == 1935
# Stat
sdss_dr7_stat = LLSSurvey.load_SDSS_DR7()
assert len(sdss_dr7_stat.NHI) == 254
def test_sdss():
""" Test SDSS DR7 -- This is very slow..
"""
# All
sdss_dr7_all = LLSSurvey.load_SDSS_DR7(sample='all')
assert sdss_dr7_all.nsys == 1935
# Stat
sdss_dr7_stat = LLSSurvey.load_SDSS_DR7()
assert len(sdss_dr7_stat.NHI) == 254
def test_z3mage():
""" Test z~3 MagE
"""
# All
z3mage = LLSSurvey.load_mage_z3()
assert z3mage.nsys == 60
assert len(z3mage.sightlines) == 105
# Non-Color
z3mage_NC = LLSSurvey.load_mage_z3(sample='non-color')
assert z3mage_NC.nsys == 32
assert len(z3mage_NC.sightlines) == 61
def test_z3mage():
""" Test z~3 MagE
"""
# All
z3mage = LLSSurvey.load_mage_z3()
assert z3mage.nsys == 60
assert len(z3mage.sightlines) == 105
# Non-Color
z3mage_NC = LLSSurvey.load_mage_z3(sample='non-color')
assert z3mage_NC.nsys == 32
assert len(z3mage_NC.sightlines) == 61
def test_sdss():
""" Test SDSS DR7 -- This is very slow..
"""
# All
sdss_dr7_all = LLSSurvey.load_SDSS_DR7(sample='all')
assert sdss_dr7_all.nsys == 1935
# Stat
sdss_dr7_stat = LLSSurvey.load_SDSS_DR7()
assert len(sdss_dr7_stat.NHI) == 191
#
z_bins = np.array([3.5, 3.65, 3.9, 4.1, 4.4])
lz, sig_lz_low, sig_lz_up = sdss_dr7_stat.binned_loz(z_bins,
NHI_mnx=(17.49, 23.))
def test_hst():
""" Test HST surveys
"""
# ACS
acs = LLSSurvey.load_HST_ACS()
assert acs.nsys == 34
assert len(acs.sightlines) == 18
# WFC3
wfc3 = LLSSurvey.load_HST_WFC3()
assert wfc3.nsys == 91
assert len(wfc3.sightlines) == 53
# Combined
HST_LLS = wfc3 + acs
assert HST_LLS.nsys == 125
assert len(HST_LLS.sightlines) == 71
def test_hst():
""" Test HST surveys
"""
# ACS
acs = LLSSurvey.load_HST_ACS()
assert acs.nsys == 34
assert len(acs.sightlines) == 18
# WFC3
wfc3 = LLSSurvey.load_HST_WFC3()
assert wfc3.nsys == 91
assert len(wfc3.sightlines) == 53
# Combined
HST_LLS = wfc3 + acs
assert HST_LLS.nsys == 125
assert len(HST_LLS.sightlines) == 71
def test_z3mage():
""" Test z~3 MagE
"""
# All
z3mage = LLSSurvey.load_mage_z3()
assert z3mage.nsys == 60
assert len(z3mage.sightlines) == 105
# Non-Color
z3mage_NC = LLSSurvey.load_mage_z3(sample='non-color')
assert z3mage_NC.nsys == 27
assert len(z3mage_NC.sightlines) == 61
# Stats
lz, sig_lz_low, sig_lz_up = z3mage_NC.binned_loz([2.6, 3.],
NHI_mnx=(17.49, 23.))
assert np.isclose(lz[0], 1.20521669)
def test_read_hdlls_dr1(): # This might actually be local now..
hdlls = LLSSurvey.load_HDLLS()
assert hdlls.nsys == 157
CII_clms = hdlls.ions((6,2))
gdCII = np.where(CII_clms['flag_N']>0)[0]
assert len(gdCII) == 103
def test_read_hdlls_dr1():
hdlls = LLSSurvey.load_HDLLS()
assert hdlls.nsys == 157
CII_clms = hdlls.ions((6,2))
gdCII = np.where(CII_clms['flag_N']>0)[0]
assert len(gdCII) == 103
def test_read_hdlls_dr1(): # This might actually be local now..
hdlls = LLSSurvey.load_HDLLS()
assert hdlls.nsys == 157
CII_clms = hdlls.ions((6, 2))
gdCII = np.where(CII_clms['flag_N'] > 0)[0]
assert len(gdCII) == 103
def test_hst():
""" Test HST surveys
"""
# ACS
acs = LLSSurvey.load_HST_ACS()
assert acs.nsys == 9
assert len(acs.sightlines) == 18
# WFC3
wfc3 = LLSSurvey.load_HST_WFC3()
#assert wfc3.nsys == 91
assert wfc3.nsys == 32 # Was 30. I think the NHI = 17.49 mattered
assert len(wfc3.sightlines) == 53
# Combined
HST_LLS = wfc3 + acs
#assert HST_LLS.nsys == 125
assert HST_LLS.nsys == 41 # Was 39
assert len(HST_LLS.sightlines) == 71
def test_load_ribaudo11():
ribaudo11 = LLSSurvey.load_ribaudo()
z, gz = ribaudo13.calculate_gz()
assert gz[0] == 1
assert gz[-1] == 3
assert ribaudo11.nsys == 50
# Stats
lz, sig_lz_low, sig_lz_up = ribaudo11.binned_loz(
[0.242, 1.078, 1.544, 1.947], NHI_mnx=(17.49, 23.))
def test_dat_list():
"""JXP format :: Likely to be Deprecated
"""
# LLS Survey
if os.getenv('LLSTREE') is None:
assert True
return
# Load
lls = LLSSurvey.from_flist('Lists/lls_metals.lst', tree=os.getenv('LLSTREE'))
# tests
np.testing.assert_allclose(lls.NHI[0], 19.25)
assert lls.nsys == 165
def test_dat_list():
"""JXP format :: Likely to be Deprecated
"""
# LLS Survey
if os.getenv('LLSTREE') is None:
assert True
return
# Load
lls = LLSSurvey.from_flist('Lists/lls_metals.lst',
tree=os.getenv('LLSTREE'))
# tests
np.testing.assert_allclose(lls.NHI[0], 19.25)
assert lls.nsys == 165
def test_gz():
# All
z3mage = LLSSurvey.load_mage_z3()
zeval, gz = z3mage.calculate_gz()
assert gz[4000] == 67
np.testing.assert_allclose(zeval[4000], 2.8705998897560931)
def json_to_sdss_dlasurvey(json_file, sdss_survey, add_pf=True, debug=False):
""" Convert JSON output file to a DLASurvey object
Assumes SDSS bookkeeping for sightlines (i.e. PLATE, FIBER)
Parameters
----------
json_file : str
Full path to the JSON results file
sdss_survey : DLASurvey
SDSS survey, usually human (e.g. JXP for DR5)
add_pf : bool, optional
Add plate/fiber to DLAs in sdss_survey
Returns
-------
ml_survey : LLSSurvey
Survey object for the LLS
"""
print("Loading SDSS Survey from JSON file {:s}".format(json_file))
# imports
from pyigm.abssys.dla import DLASystem
from pyigm.abssys.lls import LLSSystem
# Fiber key
for fkey in ['FIBER', 'FIBER_ID', 'FIB']:
if fkey in sdss_survey.sightlines.keys():
break
# Read
ml_results = ltu.loadjson(json_file)
use_platef = False
if 'plate' in ml_results[0].keys():
use_platef = True
else:
if 'id' in ml_results[0].keys():
use_id = True
# Init
#idict = dict(plate=[], fiber=[], classification_confidence=[], # FOR v2
# classification=[], ra=[], dec=[])
idict = dict(ra=[], dec=[])
if use_platef:
for key in ['plate', 'fiber', 'mjd']:
idict[key] = []
ml_tbl = Table()
ml_survey = LLSSurvey()
systems = []
in_ml = np.array([False]*len(sdss_survey.sightlines))
# Loop
for obj in ml_results:
# Sightline
for key in idict.keys():
idict[key].append(obj[key])
# DLAs
#if debug:
# if (obj['plate'] == 1366) & (obj['fiber'] == 614):
# sv_coord = SkyCoord(ra=obj['ra'], dec=obj['dec'], unit='deg')
# print("GOT A MATCH IN RESULTS FILE")
for idla in obj['dlas']:
"""
dla = DLASystem((sdss_survey.sightlines['RA'][mt[0]],
sdss_survey.sightlines['DEC'][mt[0]]),
idla['spectrum']/(1215.6701)-1., None,
idla['column_density'])
"""
if idla['z_dla'] < 1.8:
continue
isys = LLSSystem((obj['ra'],obj['dec']),
idla['z_dla'], None, NHI=idla['column_density'], zem=obj['z_qso'])
isys.confidence = idla['dla_confidence']
if use_platef:
isys.plate = obj['plate']
isys.fiber = obj['fiber']
elif use_id:
plate, fiber = [int(spl) for spl in obj['id'].split('-')]
isys.plate = plate
isys.fiber = fiber
# Save
systems.append(isys)
# Connect to sightlines
ml_coord = SkyCoord(ra=idict['ra'], dec=idict['dec'], unit='deg')
s_coord = SkyCoord(ra=sdss_survey.sightlines['RA'], dec=sdss_survey.sightlines['DEC'], unit='deg')
idx, d2d, d3d = match_coordinates_sky(s_coord, ml_coord, nthneighbor=1)
used = d2d < 1.*u.arcsec
for iidx in np.where(~used)[0]:
print("Sightline RA={:g}, DEC={:g} was not used".format(sdss_survey.sightlines['RA'][iidx],
sdss_survey.sightlines['DEC'][iidx]))
# Add plate/fiber to statistical DLAs
if add_pf:
dla_coord = sdss_survey.coord
idx2, d2d, d3d = match_coordinates_sky(dla_coord, s_coord, nthneighbor=1)
if np.min(d2d.to('arcsec').value) > 1.:
raise ValueError("Bad match to sightlines")
for jj,igd in enumerate(np.where(sdss_survey.mask)[0]):
dla = sdss_survey._abs_sys[igd]
try:
dla.plate = sdss_survey.sightlines['PLATE'][idx2[jj]]
except IndexError:
pdb.set_trace()
dla.fiber = sdss_survey.sightlines[fkey][idx2[jj]]
# Finish
ml_survey._abs_sys = systems
if debug:
ml2_coord = ml_survey.coord
minsep = np.min(sv_coord.separation(ml2_coord))
minsep2 = np.min(sv_coord.separation(s_coord))
tmp = sdss_survey.sightlines[used]
t_coord = SkyCoord(ra=tmp['RA'], dec=tmp['DEC'], unit='deg')
minsep3 = np.min(sv_coord.separation(t_coord))
pdb.set_trace()
ml_survey.sightlines = sdss_survey.sightlines[used]
for key in idict.keys():
ml_tbl[key] = idict[key]
ml_survey.ml_tbl = ml_tbl
# Return
return ml_survey
def test_gz():
# All
z3mage = LLSSurvey.load_mage_z3()
zeval, gz = z3mage.calculate_gz()
assert gz[4000] == 67
np.testing.assert_allclose(zeval[4000], 2.8705998897560931)
def test_read_hdlls_dr1_simple():
hdlls = LLSSurvey.load_HDLLS(load_sys=False)
assert hdlls.nsys == 157
def load_ml_file(pred_file):
""" Load the search results from the CNN into a DLASurvey object
Parameters
----------
pred_file
Returns
-------
ml_llssurvey: LLSSurvey
ml_dlasusrvey: DLASurvey
"""
print("Loading {:s}. Please be patient..".format(pred_file))
# Read
ml_results = ltu.loadjson(pred_file)
use_platef = False
if 'plate' in ml_results[0].keys():
use_platef = True
else:
if 'id' in ml_results[0].keys():
use_id = True
# Init
idict = dict(ra=[], dec=[], plate=[], fiber=[])
if use_platef:
for key in ['plate', 'fiber', 'mjd']:
idict[key] = []
dlasystems = []
llssystems = []
# Generate coords to speed things up
for obj in ml_results:
for key in ['ra', 'dec']:
idict[key].append(obj[key])
ml_coords = SkyCoord(ra=idict['ra'], dec=idict['dec'], unit='deg')
ra_names = ml_coords.icrs.ra.to_string(unit=u.hour, sep='', pad=True)
dec_names = ml_coords.icrs.dec.to_string(sep='', pad=True, alwayssign=True)
vlim = [-500., 500.] * u.km / u.s
dcoord = SkyCoord(ra=0., dec=0., unit='deg')
# Loop on list
didx, lidx = [], []
print("Looping on sightlines..")
for tt, obj in enumerate(ml_results):
#if (tt % 100) == 0:
# print('tt: {:d}'.format(tt))
# Sightline
if use_id:
plate, fiber = [int(spl) for spl in obj['id'].split('-')]
idict['plate'].append(plate)
idict['fiber'].append(fiber)
# Systems
for ss, syskey in enumerate(['dlas', 'subdlas']):
for idla in obj[syskey]:
name = 'J{:s}{:s}_z{:.3f}'.format(ra_names[tt], dec_names[tt],
idla['z_dla'])
if ss == 0:
isys = DLASystem(dcoord,
idla['z_dla'],
vlim,
NHI=idla['column_density'],
zem=obj['z_qso'],
name=name)
else:
isys = LLSSystem(dcoord,
idla['z_dla'],
vlim,
NHI=idla['column_density'],
zem=obj['z_qso'],
name=name)
isys.confidence = idla['dla_confidence']
isys.s2n = idla['s2n']
if use_platef:
isys.plate = obj['plate']
isys.fiber = obj['fiber']
elif use_id:
isys.plate = plate
isys.fiber = fiber
# Save
if ss == 0:
didx.append(tt)
dlasystems.append(isys)
else:
lidx.append(tt)
llssystems.append(isys)
# Generate sightline tables
sightlines = Table()
sightlines['RA'] = idict['ra']
sightlines['DEC'] = idict['dec']
sightlines['PLATE'] = idict['plate']
sightlines['FIBERID'] = idict['fiber']
# Surveys
ml_llssurvey = LLSSurvey()
ml_llssurvey.sightlines = sightlines.copy()
ml_llssurvey._abs_sys = llssystems
ml_llssurvey.coords = ml_coords[np.array(lidx)]
ml_dlasurvey = DLASurvey()
ml_dlasurvey.sightlines = sightlines.copy()
ml_dlasurvey._abs_sys = dlasystems
ml_dlasurvey.coords = ml_coords[np.array(didx)]
# Return
return ml_llssurvey, ml_dlasurvey
def test_read_hdlls_dr1_simple():
hdlls = LLSSurvey.load_HDLLS(load_sys=False)
assert hdlls.nsys == 157