def main(args=None):
from astropy.coordinates import SkyCoord
from astropy import units as u
from linetools import utils as ltu
from pyigm.abssys.dla import DLASystem
from pyigm.abssys.lls import LLSSystem
if args is None:
pargs = parser()
else:
pargs = args
# Coordinates
if pargs.jcoord is not None:
coord = ltu.radec_to_coord(pargs.jcoord)
else:
coord = SkyCoord(ra=0., dec=0., unit='deg')
# vlim
if pargs.vlim is not None:
vlims = [float(vlim) for vlim in pargs.vlim.split(',')]*u.km/u.s
else:
vlims = None
# go
if pargs.itype == 'dla':
isys = DLASystem(coord, pargs.zabs, vlims, pargs.NHI, zem=pargs.zem, sig_NHI=pargs.sigNHI)
elif pargs.itype == 'lls':
isys = LLSSystem(coord, pargs.zabs, vlims, NHI=pargs.NHI, zem=pargs.zem, sig_NHI=pargs.sigNHI)
else:
raise IOError("Not prepared for this type of IGMSystem")
# Write
isys.write_json(pargs.outfile)
def add_DLA(self,z, NHI=20.3,bval=30.*u.km/u.s, comment='None', model=True):
"""Generate a new DLA
"""
# Lya, Lyb
dla_lines = [] # For convenience
for trans in ['HI 1025', 'HI 1215']:
iline = AbsLine(trans)
iline.attrib['z'] = z
iline.attrib['N'] = 10**NHI / u.cm**2
iline.attrib['b'] = bval
iline.attrib['coord'] = SkyCoord(ra=0*u.deg,dec=0*u.deg)
dla_lines.append(iline)
# Generate system
new_sys = DLASystem.from_abslines(dla_lines) #(0*u.deg,0*u.deg),z,None,NHI)
new_sys.bval = bval # This is not standard, but for convenience
new_sys.comment = comment
new_sys.dla_lines = dla_lines # Also for convenience
# Name
self.count_dla += 1
new_sys.label = 'DLA_Sys_{:d}'.format(self.count_dla)
# Add
self.abssys_widg.add_fil(new_sys.label)
self.abssys_widg.all_abssys.append(new_sys)
self.abssys_widg.abslist_widget.item(
len(self.abssys_widg.all_abssys)).setSelected(True)
# Update
self.llist['Plot'] = False # Turn off metal-lines
if model: # For dealing with initialization
self.update_model()
def profile():
coord = SkyCoord(ra=12.231, dec=-12.2432, unit='deg')
dla_list = []
vlim = [-500., 500.]*u.km/u.s
for ii in range(10000):
if (ii % 100) == 0:
print('tt: {:d}'.format(ii))
isys = DLASystem(coord, 2., vlim, NHI=21.0, zem=3., name='dumb')
dla_list.append(isys)
return None
def test_model_abs():
# Simple system (without an absline)
dla = DLASystem.from_json(data_path('J010311.38+131616.7_z2.309_ESI.json'))
spec_fil = linetools.__path__[0]+'/spectra/tests/files/PH957_f.fits'
spec = lsio.readspec(spec_fil)
model, lya_lines = dla.model_abs(spec)
# import pdb; pdb.set_trace()
# Check core
ipx = np.argmin(np.abs(spec.wavelength.value-(1+dla.zabs)*1215.67))
assert model.flux[ipx].value < 1e-4
def test_dat_init():
# JXP .dat files
if os.getenv('DLA') is None:
assert True
return
# Read
datfil = 'Data/PH957.z2309.dat'
dla = DLASystem.from_datfile(datfil, tree=os.environ.get('DLA'))
#
np.testing.assert_allclose(dla.NHI, 21.37)
np.testing.assert_allclose(dla.zabs, 2.309)
def insert_dlas(sightline,
overlap=False,
rstate=None,
slls=False,
mix=False,
high=False,
noise=False):
""" Insert a DLA into input spectrum
Also adjusts the noise
Will also add noise 'everywhere' if requested
Parameters
----------
sightline:dla_cnn.data_model.sightline.Sightline object
overlap: bool
noise: bool, optional
Returns
-------
None
"""
#init
if rstate is None:
rstate = np.random.RandomState()
spec = XSpectrum1D.from_tuple(
(10**sightline.loglam, sightline.flux)) #generate xspectrum1d
# Generate DLAs
dlas = []
spec_dlas = []
zabslist = init_zabs(sightline, overlap)
for zabs in zabslist:
# Random NHI
NHI = uniform_NHI(slls=slls, mix=mix, high=high)
spec_dla = Dla((1 + zabs) * 1215.6701, NHI, '00' + str(jj))
if (slls or mix):
dla = LLSSystem((sightline.ra, sightline.dec), zabs, None, NHI=NHI)
else:
dla = DLASystem((sightline.ra, sightline.dec), zabs, None, NHI)
dlas.append(dla)
spec_dlas.append(spec_dla)
# Insert dlas to one sightline
vmodel, _ = hi_model(dlas, spec, fwhm=3.)
#add noise
if noise:
rand = rstate.randn(len(sightline.flux))
noise = rand * sightline.error * np.sqrt(1 - vmodel.flux.value**2)
else:
noise = 0
final_spec = XSpectrum1D.from_tuple(
(vmodel.wavelength, spec.flux.value * vmodel.flux.value + noise))
#generate new sightline
sightline.flux = final_spec.flux.value
sightline.dlas = spec_dlas
sightline.s2n = estimate_s2n(sightline)
def test_dla_XY():
spec_fil = linetools.__path__[0]+'/spectra/tests/files/PH957_f.fits'
spec = lsio.readspec(spec_fil)
dla = DLASystem.from_json(data_path('J010311.38+131616.7_z2.309_ESI.json'))
#
dla.measure_aodm(spec=spec)
dla.update_component_colm()
dla.fill_ionN()
dla.XY = RelAbund.from_ionclm_table((1,21.37,0.08), dla._ionN)
tbl = dla.XY.table()
assert len(tbl) == 8
def test_parse_ion():
# JXP .ion file
if os.getenv('DLA') is None:
assert True
return
# Read
datfil = 'Data/PH957.z2309.dat'
dla = DLASystem.from_datfile(datfil, tree=os.environ.get('DLA'))
#
dla.get_ions(use_Nfile=True)
assert len(dla._ionN) == 14
def test_dat_init():
# JXP .dat files
if os.getenv('DLA') is None:
assert True
return
# Read
datfil = 'Data/PH957.z2309.dat'
dla = DLASystem.from_datfile(datfil, tree=os.environ.get('DLA'))
#
np.testing.assert_allclose(dla.NHI, 21.37)
np.testing.assert_allclose(dla.zabs, 2.309)
def test_parse_ion():
# JXP .ion file
if os.getenv('DLA') is None:
assert True
return
# Read
datfil = 'Data/PH957.z2309.dat'
dla = DLASystem.from_datfile(datfil, tree=os.environ.get('DLA'))
#
dla.get_ions(use_Nfile=True)
assert len(dla._ionN) == 13
def test_dla_XY():
spec_fil = linetools.__path__[0]+'/spectra/tests/files/PH957_f.fits'
spec = lsio.readspec(spec_fil)
dla = DLASystem.from_json(data_path('J010311.38+131616.7_z2.309_ESI.json'))
#
dla.measure_aodm(spec=spec)
dla.update_component_colm()
dla.fill_ionN()
dla.XY = RelAbund.from_ionclm_table((1,21.37,0.08), dla._ionN)
tbl = dla.XY.table()
assert len(tbl) == 8
def test_mkigmsys():
outfile = 'tmp.json'
pargs = pyigm_mkigmsys.parser(
['dla', '3.0', outfile, '--NHI=20.5', '--zem=4.', '--vlim=-232,300'])
pyigm_mkigmsys.main(args=pargs)
# Read as JSON
with open(outfile, 'r') as f:
jdict = json.load(f)
assert np.isclose(jdict['zabs'], 3.)
# Now load as a system
dla = DLASystem.from_dict(jdict)
assert np.isclose(dla.NHI, 20.5)
def get_dla(zabs, NHI, matrix_lam, matrix_flux, wvoff=60.):
spec = XSpectrum1D.from_tuple((matrix_lam, matrix_flux))
if NHI < 20.3:
dla = LLSSystem((0, 0), zabs, None, NHI=NHI)
else:
dla = DLASystem((0, 0), zabs, None, NHI)
wvcen = (1 + zabs) * 1215.67
gd_wv = (spec.wavelength.value >
wvcen - wvoff - 30) & (spec.wavelength.value < wvcen + wvoff + 30)
co = 1.5 #np.mean(spec.flux[gd_wv])#amax
lya, lines = hi_model(dla, spec, lya_only=True)
return lya.wavelength[gd_wv], co * lya.flux[gd_wv]
def main(args=None):
from linetools import utils as ltu
pargs = parser()
# Read
jdict = ltu.loadjson(pargs.jsonfile)
if 'class' not in jdict.keys():
raise KeyError("This script only works with JSON files with named classes")
if jdict['class'] == 'IGMSightline':
from pyigm.igm.igmsightline import IGMSightline
obj = IGMSightline.from_dict(jdict)
flg_tbl = True
elif jdict['class'] == 'DLASystem':
from pyigm.abssys.dla import DLASystem
obj = DLASystem.from_dict(jdict)
flg_tbl = False
elif jdict['class'] == 'LLSSystem':
from pyigm.abssys.lls import LLSSystem
obj = LLSSystem.from_dict(jdict)
obj.fill_ionN()
flg_tbl = True # Column density table
else:
raise IOError("Not prepared for this class: {:s}".format(jdict['class']))
# name
try:
name = jdict['name']
except KeyError:
try:
name = jdict['Name']
except:
name = 'None'
print("Name of object: {:s}".format(name))
# Generate table
if flg_tbl:
if jdict['class'] == 'IGMSightline':
tbl = obj.build_table()
elif jdict['class'] == 'LLSSystem':
tbl = obj._ionN
tbl['logN'].format = '5.2f'
tbl['sig_logN'].format = '5.2f'
tbl['vmin'].format = '8.1f'
tbl['vmax'].format = '8.1f'
else:
tbl = None
# Print
if len(tbl) > 0:
tbl.pprint(99999, max_width=120)
else:
print("Table was empty..")
def add_DLA(self,
z,
NHI=20.3,
bval=30. * u.km / u.s,
comment='None',
model=True):
"""Generate a new DLA
"""
# Lya, Lyb
dla_lines = [] # For convenience
for trans in ['HI 1025', 'HI 1215']:
iline = AbsLine(trans, z=z)
iline.attrib['flag_N'] = 1
iline.attrib['N'] = 10**NHI / u.cm**2
iline.attrib['sig_N'] = 1 / u.cm**2 # Avoid nan
iline.attrib['b'] = bval
iline.attrib['coord'] = SkyCoord(ra=0 * u.deg, dec=0 * u.deg)
dla_lines.append(iline)
# Generate system
HIcomponent = ltiu.build_components_from_abslines(dla_lines)[0]
HIcomponent.synthesize_colm()
new_sys = DLASystem.from_components([HIcomponent
]) #(0*u.deg,0*u.deg),z,None,NHI)
#QtCore.pyqtRemoveInputHook()
#import pdb; pdb.set_trace()
#QtCore.pyqtRestoreInputHook()
new_sys.bval = bval # This is not standard, but for convenience
new_sys.comment = comment
new_sys.dla_lines = dla_lines # Also for convenience
# Name
self.count_dla += 1
new_sys.label = 'DLA_Sys_{:d}'.format(self.count_dla)
# Add
self.abssys_widg.add_fil(new_sys.label)
self.abssys_widg.all_abssys.append(new_sys)
self.abssys_widg.abslist_widget.item(len(
self.abssys_widg.all_abssys)).setSelected(True)
# Update
self.llist['Plot'] = False # Turn off metal-lines
if model: # For dealing with initialization
self.update_model()
def test_DLA_from_components():
radec = SkyCoord(ra=123.1143*u.deg, dec=-12.4321*u.deg)
# HI Lya, Lyb
lya = AbsLine(1215.670*u.AA)
lya.analy['vlim'] = [-300.,300.]*u.km/u.s
lya.attrib['z'] = 2.92939
lya.attrib['N'] = 3e20 / u.cm**2
lyb = AbsLine(1025.7222*u.AA)
lyb.analy['vlim'] = [-300.,300.]*u.km/u.s
lyb.attrib['z'] = lya.attrib['z']
lyb.attrib['N'] = 3e20 / u.cm**2
abscomp = AbsComponent.from_abslines([lya,lyb])
abscomp.coord = radec
# Instantiate
HIsys = DLASystem.from_components([abscomp])
# Test
np.testing.assert_allclose(HIsys.NHI, 20.477121254719663)
assert len(HIsys._components) == 1
assert HIsys._components[0].Zion[0] == 1
assert HIsys._components[0].Zion[1] == 1
def test_init():
dlas = DLASurvey(ref='null')
assert dlas.abs_type == 'DLA'
coord = SkyCoord(ra=123.1143, dec=-12.4321, unit='deg')
dlasys = DLASystem(coord, 1.244, [-300, 300.] * u.km / u.s, 20.4)
dlasys.name = 'Sys1'
#
coord2 = SkyCoord(ra=223.1143, dec=42.4321, unit='deg')
dlasys2 = DLASystem(coord2, 1.744, [-300, 300.] * u.km / u.s, 21.7)
dlasys2.name = 'Sys2'
# Add systems
dlas.add_abs_sys(dlasys)
dlas.add_abs_sys(dlasys2)
assert dlas.nsys == 2
def test_DLA_from_components():
radec = SkyCoord(ra=123.1143*u.deg, dec=-12.4321*u.deg)
# HI Lya, Lyb
lya = AbsLine(1215.670*u.AA, z=2.92939)
lya.analy['vlim'] = [-300.,300.]*u.km/u.s
lya.attrib['flag_N'] = 1
lya.attrib['N'] = 3e20 / u.cm**2
lya.attrib['sig_N'] = [1]*2 / u.cm**2
lyb = AbsLine(1025.7222*u.AA, z=lya.z)
lyb.analy['vlim'] = [-300.,300.]*u.km/u.s
lyb.attrib['N'] = 3e20 / u.cm**2
lyb.attrib['flag_N'] = 1
lyb.attrib['sig_N'] = [1]*2 / u.cm**2
abscomp = AbsComponent.from_abslines([lya,lyb])
abscomp.coord = radec
# Instantiate
HIsys = DLASystem.from_components([abscomp])
# Test
np.testing.assert_allclose(HIsys.NHI, 20.477121254719663)
assert len(HIsys._components) == 1
assert HIsys._components[0].Zion[0] == 1
assert HIsys._components[0].Zion[1] == 1
def add_DLA(self,
z,
NHI=20.3,
bval=30. * u.km / u.s,
comment='None',
model=True):
"""Generate a new DLA
"""
# Lya, Lyb
dla_lines = [] # For convenience
for trans in ['HI 1025', 'HI 1215']:
iline = AbsLine(trans)
iline.attrib['z'] = z
iline.attrib['N'] = 10**NHI / u.cm**2
iline.attrib['b'] = bval
iline.attrib['coord'] = SkyCoord(ra=0 * u.deg, dec=0 * u.deg)
dla_lines.append(iline)
# Generate system
new_sys = DLASystem.from_abslines(
dla_lines) #(0*u.deg,0*u.deg),z,None,NHI)
new_sys.bval = bval # This is not standard, but for convenience
new_sys.comment = comment
new_sys.dla_lines = dla_lines # Also for convenience
# Name
self.count_dla += 1
new_sys.label = 'DLA_Sys_{:d}'.format(self.count_dla)
# Add
self.abssys_widg.add_fil(new_sys.label)
self.abssys_widg.all_abssys.append(new_sys)
self.abssys_widg.abslist_widget.item(len(
self.abssys_widg.all_abssys)).setSelected(True)
# Update
self.llist['Plot'] = False # Turn off metal-lines
if model: # For dealing with initialization
self.update_model()
def insert_dlas(spec,
zem,
fNHI=None,
rstate=None,
slls=False,
mix=False,
high=False,
low_s2n=False,
noise_boost=4.):
""" Insert a DLA into input spectrum
Also adjusts the noise
Will also add noise 'everywhere' if requested
Parameters
----------
spec
fNHI
rstate
low_s2n : bool, optional
Reduce the S/N everywhere. By a factor of noise_boost
noise_boost : float, optional
Factor to *increase* the noise by
Returns
-------
final_spec : XSpectrum1D
dlas : list
List of DLAs inserted
"""
from pyigm.fN import dla as pyi_fd
from pyigm.abssys.dla import DLASystem
from pyigm.abssys.lls import LLSSystem
from pyigm.abssys.utils import hi_model
# Init
if rstate is None:
rstate = np.random.RandomState()
if fNHI is None:
fNHI = init_fNHI(slls=slls, mix=mix, high=high)
# Allowed redshift placement
## Cut on zem and 910A rest-frame
zlya = spec.wavelength.value / 1215.67 - 1
dz = np.roll(zlya, -1) - zlya
dz[-1] = dz[-2]
gdz = (zlya < zem) & (spec.wavelength > 910. * u.AA * (1 + zem))
# l(z) -- Uses DLA for SLLS too which is fine
lz = pyi_fd.lX(zlya[gdz], extrap=True, calc_lz=True)
cum_lz = np.cumsum(lz * dz[gdz])
tot_lz = cum_lz[-1]
fzdla = interpolate.interp1d(cum_lz / tot_lz,
zlya[gdz],
bounds_error=False,
fill_value=np.min(zlya[gdz])) #
# n DLA
nDLA = 0
while nDLA == 0:
nval = rstate.poisson(tot_lz, 100)
gdv = nval > 0
if np.sum(gdv) == 0:
continue
else:
nDLA = nval[np.where(gdv)[0][0]]
# Generate DLAs
dlas = []
for jj in range(nDLA):
# Random z
zabs = float(fzdla(rstate.random_sample()))
# Random NHI
NHI = float(fNHI(rstate.random_sample()))
if (slls or mix):
dla = LLSSystem((0., 0), zabs, None, NHI=NHI)
else:
dla = DLASystem((0., 0), zabs, (None, None), NHI)
dlas.append(dla)
# Insert
vmodel, _ = hi_model(dlas, spec, fwhm=3., llist=llist)
# Add noise
rand = rstate.randn(spec.npix)
noise = rand * spec.sig * (1 - vmodel.flux.value)
# More noise??
if low_s2n:
rand2 = rstate.randn(spec.npix)
more_noise = noise_boost * rand2 * spec.sig
noise += more_noise
else:
s2n_boost = 1.
final_spec = XSpectrum1D.from_tuple(
(vmodel.wavelength, spec.flux.value * vmodel.flux.value + noise,
noise_boost * spec.sig))
# Return
return final_spec, dlas
def test_dla_from_dict():
dla = DLASystem.from_json(data_path('J010311.38+131616.7_z2.309_ESI.json'))
assert len(dla._components) == 16
def load_ml_file(pred_file):
""" Load the search results from the CNN into a DLASurvey object
Parameters
----------
pred_file
Returns
-------
ml_llssurvey: LLSSurvey
ml_dlasusrvey: DLASurvey
"""
print("Loading {:s}. Please be patient..".format(pred_file))
# Read
ml_results = ltu.loadjson(pred_file)
use_platef = False
if 'plate' in ml_results[0].keys():
use_platef = True
else:
if 'id' in ml_results[0].keys():
use_id = True
# Init
idict = dict(ra=[], dec=[], plate=[], fiber=[])
if use_platef:
for key in ['plate', 'fiber', 'mjd']:
idict[key] = []
dlasystems = []
llssystems = []
# Generate coords to speed things up
for obj in ml_results:
for key in ['ra', 'dec']:
idict[key].append(obj[key])
ml_coords = SkyCoord(ra=idict['ra'], dec=idict['dec'], unit='deg')
ra_names = ml_coords.icrs.ra.to_string(unit=u.hour, sep='', pad=True)
dec_names = ml_coords.icrs.dec.to_string(sep='', pad=True, alwayssign=True)
vlim = [-500., 500.] * u.km / u.s
dcoord = SkyCoord(ra=0., dec=0., unit='deg')
# Loop on list
didx, lidx = [], []
print("Looping on sightlines..")
for tt, obj in enumerate(ml_results):
#if (tt % 100) == 0:
# print('tt: {:d}'.format(tt))
# Sightline
if use_id:
plate, fiber = [int(spl) for spl in obj['id'].split('-')]
idict['plate'].append(plate)
idict['fiber'].append(fiber)
# Systems
for ss, syskey in enumerate(['dlas', 'subdlas']):
for idla in obj[syskey]:
name = 'J{:s}{:s}_z{:.3f}'.format(ra_names[tt], dec_names[tt],
idla['z_dla'])
if ss == 0:
isys = DLASystem(dcoord,
idla['z_dla'],
vlim,
NHI=idla['column_density'],
zem=obj['z_qso'],
name=name)
else:
isys = LLSSystem(dcoord,
idla['z_dla'],
vlim,
NHI=idla['column_density'],
zem=obj['z_qso'],
name=name)
isys.confidence = idla['dla_confidence']
isys.s2n = idla['s2n']
if use_platef:
isys.plate = obj['plate']
isys.fiber = obj['fiber']
elif use_id:
isys.plate = plate
isys.fiber = fiber
# Save
if ss == 0:
didx.append(tt)
dlasystems.append(isys)
else:
lidx.append(tt)
llssystems.append(isys)
# Generate sightline tables
sightlines = Table()
sightlines['RA'] = idict['ra']
sightlines['DEC'] = idict['dec']
sightlines['PLATE'] = idict['plate']
sightlines['FIBERID'] = idict['fiber']
# Surveys
ml_llssurvey = LLSSurvey()
ml_llssurvey.sightlines = sightlines.copy()
ml_llssurvey._abs_sys = llssystems
ml_llssurvey.coords = ml_coords[np.array(lidx)]
ml_dlasurvey = DLASurvey()
ml_dlasurvey.sightlines = sightlines.copy()
ml_dlasurvey._abs_sys = dlasystems
ml_dlasurvey.coords = ml_coords[np.array(didx)]
# Return
return ml_llssurvey, ml_dlasurvey
def test_dla_from_dict():
dla = DLASystem.from_json(data_path('J010311.38+131616.7_z2.309_ESI.json'))
assert len(dla._components) == 16
def test_simple_dla_init():
# Init
dla = DLASystem((0.*u.deg, 0.*u.deg), 2.5, None, NHI=20.55)
#
np.testing.assert_allclose(dla.vlim[0].value,-500.)
np.testing.assert_allclose(dla.NHI, 20.55)
