def test_compare_stuff():
# Compare files
assert ltu.compare_two_files(data_path('dum_file1.txt'), data_path('dum_file2.txt'))
# Compare 2 json files
assert ltu.compare_two_json(data_path('dum1.json'), data_path('dum2.json'))
# Dicts
d1 = ltu.loadjson(data_path('dum1.json'))
d3 = ltu.loadjson(data_path('dum3.json'))
added, removed, modified, same = ltu.compare_two_dict(d1,d3)
assert removed == set('e')
for key in ['a','d']:
assert key in modified.keys()
def main(args):
"""
Parameters
----------
args
Returns
-------
"""
import numpy as np
from pypit import pyputils
msgs = pyputils.get_dummy_logger()
from pypit import arqa
from linetools.utils import loadjson
# Read JSON
fdict = loadjson(args.wave_soln)
for key in fdict.keys():
if isinstance(fdict[key], list):
fdict[key] = np.array(fdict[key])
# Generate QA
arqa.arc_fit_qa(None, fdict, outfil=args.outfile, ids_only=True,
title=args.title)
print("Wrote {:s}".format(args.outfile))
def test_save_load_json():
tmp_dict = dict(a=1, b=2, c='adsf')
# Write
ltu.savejson('tmp.json', tmp_dict, overwrite=True)
# Load
new_dict = ltu.loadjson('tmp.json')
assert new_dict['a'] == 1
# Write with gzip
ltu.savejson('tmp.json.gz', tmp_dict, overwrite=True)
# Load
new_dict = ltu.loadjson('tmp.json.gz')
assert new_dict['a'] == 1
# Write with easy to read
ltu.savejson('tmp2.json', tmp_dict, overwrite=True, easy_to_read=True)
new_dict2 = ltu.loadjson('tmp2.json')
assert new_dict2['a'] == 1
def test_dicts():
# Init HI Lya
abslin = AbsLine(1215.6700*u.AA)
adict = abslin.to_dict()
assert isinstance(adict, dict)
# Write
#pdb.set_trace()
ltu.savejson('tmp.json', adict, overwrite=True)
# Read
newdict = ltu.loadjson('tmp.json')
newlin = SpectralLine.from_dict(newdict)
assert newlin.name == 'HI 1215'
def test_dicts():
# Init Halpha
emisslin = EmLine(6564.613*u.AA)
emisslin.analy['spec'] = 'tmp.fits'
edict = emisslin.to_dict()
assert isinstance(edict, dict)
# Write
ltu.savejson('tmp.json', edict, overwrite=True)
# Read
newdict = ltu.loadjson('tmp.json')
newlin = SpectralLine.from_dict(newdict)
assert newlin.name == 'Halpha'
assert newlin.ltype == 'Em'
def load_sys_files(inp, type, ref=None, sys_path=False, **kwargs):
""" Load up a set of SYS files from the hard-drive (JSON files)
Parameters
----------
inp : str
type : str
type of IGMSystem, e.g. LLS
ref : str, optional
Reference label
sys_path : str, optional
indicates that inp is a path to a set of JSON SYS files
otherwise, inp should be the filename of a tarball of JSON files
Returns
-------
survey : IGMSurvey
"""
import tarfile
#
survey = class_by_type(type)(ref=ref)
system = pyasu.class_by_type(type)
if sys_path:
# Individual files
files = glob.glob(inp+'*.json')
files.sort()
for ifile in files:
tdict = ltu.loadjson(ifile)
abssys = system.from_dict(tdict)
survey._abs_sys.append(abssys)
else: # tarball
print('Loading systems from {:s}'.format(inp))
tar = tarfile.open(inp)
for member in tar.getmembers():
if '.' not in member.name:
print('Skipping a likely folder: {:s}'.format(member.name))
continue
# Extract
f = tar.extractfile(member)
tdict = json.load(f)
# Add keys (for backwards compatability)
if ('NHI' in tdict.keys()) and ('flag_NHI' not in tdict.keys()):
tdict['flag_NHI'] = 1
# Generate
abssys = system.from_dict(tdict, chk_sep=False, **kwargs) # Consider use_coord=True as default
survey._abs_sys.append(abssys)
tar.close()
# Return
return survey
def from_json(cls, json_file, **kwargs):
"""
Parameters
----------
json_file : str
**kwargs : passed to cls.from_dict()
Returns
-------
AbsComponent
"""
# Load dict
jdict = ltu.loadjson(json_file)
# Instantiate
return cls.from_dict(jdict, **kwargs)
def from_json(cls, json_file, **kwargs):
""" Load from a JSON file (via from_dict)
Parameters
----------
json_file
kwargs
Returns
-------
AbsSystem
"""
idict = ltu.loadjson(json_file)
slf = cls.from_dict(idict, **kwargs)
return slf
def add_s2n_after(ids, json_file, debug=False, CHUNK_SIZE=1000):
from linetools import utils as ltu
from dla_cnn.absorption import get_s2n_for_absorbers # Needs to be here
# Load json file
predictions = ltu.loadjson(json_file)
jids = [ii['id'] for ii in predictions]
num_cores = multiprocessing.cpu_count() - 2
p = Pool(num_cores) # a thread pool we'll reuse
sightlines_processed_count = 0
# IDs
ids.sort(key=methodcaller('id_string'))
for sss, ids_batch in enumerate(
np.array_split(ids, np.arange(CHUNK_SIZE, len(ids), CHUNK_SIZE))):
num_sightlines = len(ids_batch)
# Read batch
process_timer = timeit.default_timer()
print("Reading {:d} sightlines with {:d} cores".format(
num_sightlines, num_cores))
sightlines_batch = p.map(read_sightline, ids_batch)
print("Done reading")
for sightline in sightlines_batch:
jidx = jids.index(sightline.id.id_string())
# Any absorbers?
if (predictions[jidx]['num_dlas']) + (
predictions[jidx]['num_subdlas']) == 0:
continue
lam, lam_rest, ix_dla_range = get_lam_data(sightline.loglam,
sightline.z_qso,
REST_RANGE)
# DLAs, subDLAs
get_s2n_for_absorbers(sightline, lam, predictions[jidx]['dlas'])
get_s2n_for_absorbers(sightline, lam, predictions[jidx]['subdlas'])
runtime = timeit.default_timer() - process_timer
print(
"Processed {:d} of {:d} in {:0.0f}s - {:0.2f}s per sample".format(
sightlines_processed_count + num_sightlines, len(ids), runtime,
runtime / num_sightlines))
sightlines_processed_count += num_sightlines
# Write
print("About to over-write your JSON file. Continue at your own risk!")
# Return new predictions
return predictions
def high_nhi_neg():
""" Examine High NHI false negatives in 10k test
"""
# Load ML
ml_abs = pred_to_tbl('../Vetting/data/test_dlas_96629_predictions.json.gz')
# Load Test
test_dlas = test_to_tbl('../Vetting/data/test_dlas_96629_10000.json.gz')
# Load vette
vette_10k = ltu.loadjson('../Vetting/vette_10k.json')
test_ml_idx = np.array(vette_10k['test_idx'])
misses = np.where(test_ml_idx == -99999)[0]
highNHI = test_dlas['NHI'][misses] > 21.2
high_tbl = test_dlas[misses[highNHI]]
# Write
high_tbl.write('test_highNHI_neg.ascii', format='ascii.fixed_width', overwrite=True)
def from_json(cls, jsonfile, **kwargs):
""" Instantiate from a JSON file
Parameters
----------
jsonfile : str
Filename
See from_dict for required keys
kwargs : passed to from_dict
Returns
-------
"""
jdict = ltu.loadjson(jsonfile)
slf = cls.from_dict(jdict, **kwargs)
# Return
return slf
def load_dla_fits(fit_file=None):
""" Load fit file(s)
Parameters
----------
fit_file : str
Returns
-------
"""
if fit_file is None:
fit_file = resource_filename('pyigm', 'data/DLA/dla_fits.json')
if os.path.exists(fit_file):
dla_fits = ltu.loadjson(fit_file)
else:
dla_fits = {}
# Return
return dla_fits, fit_file
def load_master(name, exten=0, frametype='<None>'):
"""
Load a pre-existing master calibration frame
Parameters
----------
name : str
Name of the master calibration file to be loaded
exten : int, optional
frametype : str, optional
The type of master calibration frame being loaded.
This keyword is only used for terminal print out.
Returns
-------
frame : ndarray
The data from the master calibration frame
"""
if frametype is None:
msgs.info("Loading a pre-existing master calibration frame")
try:
hdu = pyfits.open(name)
except:
msgs.error("Master calibration file does not exist:" +
msgs.newline() + name)
msgs.info("Master {0:s} frame loaded successfully:".format(
hdu[0].header['FRAMETYP']) + msgs.newline() + name)
head = hdu[0].header
data = hdu[exten].data.astype(np.float)
return data, head
#return np.array(infile[0].data, dtype=np.float)
else:
from linetools import utils as ltu
msgs.info("Loading Master {0:s} frame:".format(frametype) +
msgs.newline() + name)
if frametype == 'wv_calib':
ldict = ltu.loadjson(name)
return ldict
else:
# Load
hdu = pyfits.open(name)
head = hdu[0].header
data = hdu[exten].data.astype(np.float)
return data, head
def pred_to_tbl(pred_file):
"""
Parameters
----------
pred_file
Returns
-------
"""
spec_list = ltu.loadjson(pred_file)
ids, zabs, conf, NHI, sigNHI, biasNHI = [], [], [], [], [], []
# Loop to my loop
for ss, spec in enumerate(spec_list):
# DLAs
for dla in spec['dlas']:
if dla['type'] == "LYB":
continue
ids.append(ss)
zabs.append(dla['z_dla'])
NHI.append(dla['column_density'])
sigNHI.append(dla['std_column_density'])
biasNHI.append(dla['column_density_bias_adjust'])
conf.append(dla['dla_confidence'])
# SLLS
for slls in spec['subdlas']:
if slls['type'] == "LYB":
continue
ids.append(ss)
zabs.append(slls['z_dla'])
NHI.append(slls['column_density'])
sigNHI.append(slls['std_column_density'])
biasNHI.append(slls['column_density_bias_adjust'])
conf.append(slls['dla_confidence'])
# Table
dla_tbl = Table()
dla_tbl['ids'] = ids
dla_tbl['zabs'] = zabs
dla_tbl['conf'] = conf
dla_tbl['sigNHI'] = sigNHI
dla_tbl['biasNHI'] = biasNHI
dla_tbl['NHI'] = NHI
# Return
return dla_tbl
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("wave_soln", type=str, default=None, help="MasterWaveSoln file [JSON]")
parser.add_argument("title", type=str, default=None, help="Title for the plot")
parser.add_argument("outfile", type=str, default=None, help="Output PDF file")
pargs = parser.parse_args()
# Read JSON
fdict = loadjson(pargs.wave_soln)
for key in fdict.keys():
if isinstance(fdict[key], list):
fdict[key] = np.array(fdict[key])
# Generate QA
arqa.arc_fit_qa(None, fdict, outfil=pargs.outfile, ids_only=True, title=pargs.title)
print("Wrote {:s}".format(pargs.outfile))
def load_sys_files(inp, type, sys_path=False):
""" Load up a set of SYS files from the hard-drive (JSON files)
Parameters
----------
inp : str
type : str
type of IGMSystem
sys_path : str, optional
indicates that inp is a path to a set of SYS files
otherwise, it should be the filename of a tarball
Returns
-------
survey : IGMSurvey
"""
import tarfile
#
survey = class_by_type(type)(ref='HD-LLS')
system = pyasu.class_by_type(type)
if sys_path:
# Individual files
files = glob.glob(inp+'*.json')
files.sort()
for ifile in files:
tdict = ltu.loadjson(ifile)
abssys = system.from_dict(tdict)
survey._abs_sys.append(abssys)
else: # tarball
print('Loading systems from {:s}'.format(inp))
tar = tarfile.open(inp)
for member in tar.getmembers():
if '.' not in member.name:
print('Skipping a likely folder: {:s}'.format(member.name))
continue
# Extract
f = tar.extractfile(member)
tdict = json.load(f)
# Generate
abssys = system.from_dict(tdict)
survey._abs_sys.append(abssys)
# Return
return survey
def main(pargs, unit_test=False):
""" Shows the spectrum
"""
import sys
import pdb
from matplotlib import pyplot as plt
from linetools import utils as ltu
wvcalib = ltu.loadjson(pargs.file)
# Grab it
spec = wvcalib[pargs.slit]['spec']
plt.clf()
ax = plt.gca()
ax.plot(spec)
plt.show()
def test_dicts():
# Init HI Lya
abslin = AbsLine(1215.6700 * u.AA)
abslin.analy['spec'] = 'tmp.fits'
adict = abslin.to_dict()
assert isinstance(adict, dict)
# Write
#pdb.set_trace()
ltu.savejson('tmp.json', adict, overwrite=True)
# Read
newdict = ltu.loadjson('tmp.json')
newlin = SpectralLine.from_dict(newdict)
assert newlin.name == 'HI 1215'
# Old dict for compatability
newdict.pop('limits')
newdict['analy']['vlim'] = [-150, 150] * u.km / u.s
newdict['attrib']['z'] = 0.5
tmp3 = SpectralLine.from_dict(newdict)
assert newlin.name == 'HI 1215'
def test_dicts():
# Init HI Lya
abslin = AbsLine(1215.6700*u.AA)
abslin.analy['spec'] = 'tmp.fits'
adict = abslin.to_dict()
assert isinstance(adict, dict)
# Write
#pdb.set_trace()
ltu.savejson('tmp.json', adict, overwrite=True)
# Read
newdict = ltu.loadjson('tmp.json')
newlin = SpectralLine.from_dict(newdict)
assert newlin.name == 'HI 1215'
# Old dict for compatability
newdict.pop('limits')
newdict['analy']['vlim'] = [-150,150]*u.km/u.s
newdict['attrib']['z'] = 0.5
tmp3 = SpectralLine.from_dict(newdict)
assert newlin.name == 'HI 1215'
def pypeit_arcspec(in_file, slit):
"""
Load up the arc spectrum from an input JSON file
Args:
in_file (str):
slit (int):
slit index
Returns:
np.ndarray, np.ndarray: wave, flux
"""
wv_dict = ltu.loadjson(in_file)
iwv_calib = wv_dict[str(slit)]
x = np.arange(len(iwv_calib['spec']))
wv_vac = utils.func_val(iwv_calib['fitc'], x/iwv_calib['xnorm'], iwv_calib['function'],
minx=iwv_calib['fmin'], maxx=iwv_calib['fmax'])
# Return
return wv_vac, np.array(iwv_calib['spec']).flatten() # JXP added flatten on 2019-11-09
def test_to_tbl(test_file):
test_dict = ltu.loadjson(test_file)
ids, zabs, sl, NHI, = [], [], [], []
ntest = len(test_dict)
# Loop to my loop
for ss in range(ntest):
ndla = test_dict[str(ss)]['nDLA']
for idla in range(ndla):
ids.append(ss)
zabs.append(test_dict[str(ss)][str(idla)]['zabs'])
NHI.append(test_dict[str(ss)][str(idla)]['NHI'])
sl.append(test_dict[str(ss)]['sl'])
# Table
test_tbl = Table()
test_tbl['ids'] = ids
test_tbl['zabs'] = zabs
test_tbl['NHI'] = NHI
test_tbl['sl'] = sl
# Return
return test_tbl
def load_json(self,
jfile,
build_data=True,
build_sys=False,
verbose=True,
**kwargs):
"""
Parameters
----------
jfile : str
build_data : bool, optional
Generate the internal _data Table [very fast and recommended]
build_sys : bool, optional
Generate the list of cgm_abs objects from the internal _dict [May be slow]
kwargs
Returns
-------
"""
# Load
self._dict = ltu.loadjson(jfile)
# Generate
if build_sys:
self.build_systems_from_dict(**kwargs)
# Galaxy coords
ras = [self._dict[key]['RA'] for key in self._dict.keys()]
decs = [self._dict[key]['DEC'] for key in self._dict.keys()]
self.coords = SkyCoord(ra=ras, dec=decs, unit='deg')
# Sightline coords
ras = [self._dict[key]['igm_sys']['RA'] for key in self._dict.keys()]
decs = [self._dict[key]['igm_sys']['DEC'] for key in self._dict.keys()]
self.scoords = SkyCoord(ra=ras, dec=decs, unit='deg')
# Data table
if build_data:
self.build_data_from_dict()
# Return
return
def load_master(name, exten=0, frametype='<None>'):
"""
Load a pre-existing master calibration frame
Parameters
----------
name : str
Name of the master calibration file to be loaded
exten : int, optional
frametype : str, optional
The type of master calibration frame being loaded.
This keyword is only used for terminal print out.
Returns
-------
frame : ndarray
The data from the master calibration frame
"""
if frametype is None:
msgs.info("Loading a pre-existing master calibration frame")
try:
hdu = pyfits.open(name)
except:
msgs.error("Master calibration file does not exist:"+msgs.newline()+name)
msgs.info("Master {0:s} frame loaded successfully:".format(hdu[0].header['FRAMETYP'])+msgs.newline()+name)
head = hdu[0].header
data = hdu[exten].data.astype(np.float)
return data, head
#return np.array(infile[0].data, dtype=np.float)
else:
from linetools import utils as ltu
msgs.info("Loading Master {0:s} frame:".format(frametype)+msgs.newline()+name)
if frametype == 'wv_calib':
ldict = ltu.loadjson(name)
return ldict
else:
# Load
hdu = pyfits.open(name)
head = hdu[0].header
data = hdu[exten].data.astype(np.float)
return data, head
def main(args):
import numpy as np
from linetools.utils import loadjson
# TODO: This must be an out-dated script that is never used.
# Deprecate it?
from pypeit import arqa
from pypeit import msgs
msgs.reset(verbosity=2)
# Read JSON
fdict = loadjson(args.wave_soln)
for key in fdict.keys():
if isinstance(fdict[key], list):
fdict[key] = np.array(fdict[key])
# Generate QA
arqa.arc_fit_qa(None, fdict, outfil=args.outfile, ids_only=True,
title=args.title)
print("Wrote {:s}".format(args.outfile))
def main() :
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('wave_soln', type = str, default = None,
help = 'MasterWaveSoln file [JSON]')
parser.add_argument('title', type = str, default = None, help = 'Title for the plot')
parser.add_argument('outfile', type = str, default = None, help = 'Output PDF file')
pargs = parser.parse_args()
# Read JSON
fdict = loadjson(pargs.wave_soln)
for key in fdict.keys():
if isinstance(fdict[key], list):
fdict[key] = np.array(fdict[key])
# Generate QA
arqa.arc_fit_qa(None, fdict, outfil=pargs.outfile, ids_only=True,
title=pargs.title)
print("Wrote {:s}".format(pargs.outfile))
def main(args):
"""
Parameters
----------
args
Returns
-------
"""
import numpy as np
try:
from xastropy.xutils import xdebug as debugger
except:
import pdb as debugger
from linetools.utils import loadjson
from pypeit import arqa
from pypeit import msgs
msgs.reset(verbosity=2)
# Read JSON
fdict = loadjson(args.wave_soln)
for key in fdict.keys():
if isinstance(fdict[key], list):
fdict[key] = np.array(fdict[key])
# Generate QA
arqa.arc_fit_qa(None,
fdict,
outfil=args.outfile,
ids_only=True,
title=args.title)
print("Wrote {:s}".format(args.outfile))
def load_spectrum(spec_file, index=0):
""" Load a simple spectrum from input file
Parameters
----------
spec_file : str
.fits -- Assumes simple ndarray in 0 extension
.ascii -- Assumes Table.read(format='ascii') will work with single column
Returns
-------
"""
import h5py
iext = spec_file.rfind('.')
if 'ascii' in spec_file[iext:]:
tbl = Table.read(spec_file, format='ascii')
key = tbl.keys()[0]
spec = tbl[key].data
elif 'fits' in spec_file[iext:]:
spec = fits.open(spec_file)[0].data
elif 'hdf5' in spec_file[iext:]:
hdf = h5py.File(spec_file, 'r')
if 'arcs' in hdf.keys():
print("Taking arc={:d} in this file".format(index))
spec = hdf['arcs/' + str(index) + '/spec'].value
else:
raise IOError("Not ready for this hdf5 file")
elif 'json' in spec_file[iext:]:
jdict = ltu.loadjson(spec_file)
try:
spec = np.array(jdict['spec'])
except KeyError:
raise IOError("spec not in your JSON dict")
# Return
return spec
def from_igmguesses(cls, radec, zem, igmgfile, name=None, **kwargs):
""" Instantiate from a JSON file from IGMGuesses
The input coordinates are used for all the components
Parameters
----------
radec : RA/DEC input
See ltu.radec_to_coord for options
zem : float
Emission redshift of sightline
igmgfile : str
Filename
Returns
-------
"""
# Read
jdict = ltu.loadjson(igmgfile) # cmps, specfile
# Add in additional keys
coord = ltu.radec_to_coord(radec)
jdict['RA'] = coord.fk5.ra.deg
jdict['DEC'] = coord.fk5.dec.deg
jdict['zem'] = zem
# Name
if name is None:
name = 'J{:s}{:s}_z{:0.3f}'.format(
coord.fk5.ra.to_string(unit=u.hour, sep='', pad=True)[0:4],
coord.fk5.dec.to_string(sep='', pad=True,
alwayssign=True)[0:5], zem)
jdict['name'] = name
jdict['components'] = jdict.pop('cmps')
kwargs['use_coord'] = True
slf = cls.from_dict(jdict, **kwargs)
# Return
return slf
def examine_false_pos(test_file='data/test_dlas_96629_10000.json.gz',
pred_file='data/test_dlas_96629_predictions.json.gz',
vette_file='vette_10k.json'):
""" Examine false positives in the Test set (held out)
"""
from pyigm.surveys.dlasurvey import DLASurvey
import h5py
import json
from matplotlib import pyplot as plt
# Load Test
test_dlas = test_to_tbl(test_file)
ntest = len(test_dlas)
# Load hdf5
CNN_result_path = '/home/xavier/Projects/ML_DLA_results/CNN/'
hdf5_datafile = CNN_result_path + 'gensample_hdf5_files/test_dlas_96629_10000.hdf5'
hdf = h5py.File(hdf5_datafile, 'r')
headers = json.loads(hdf['meta'].value)['headers']
# Load ML
ml_abs = pred_to_tbl(pred_file)
# Vette
vette = ltu.loadjson(vette_file)
test_ml_idx = np.array(vette['test_idx'])
# Load DR5
dr5 = DLASurvey.load_SDSS_DR5()
all_dr5 = DLASurvey.load_SDSS_DR5(sample='all_sys')
# False positives
fpos = ml_abs['NHI'] >= 20.3 # Must be a DLA
imatched = np.where(test_ml_idx >= 0)[0]
match_val = test_ml_idx[imatched]
fpos[match_val] = False
print("There are {:d} total false positives".format(np.sum(fpos)))
# This nearly matches David's. Will run with his analysis.
fpos_in_dr5 = fpos.copy()
# Restrict on DR5
for idx in np.where(fpos_in_dr5)[0]:
# Convoluted indexing..
mlid = ml_abs['ids'][idx]
# Plate/Fiber
plate = headers[mlid]['PLATE']
fib = headers[mlid]['FIBER']
# Finally, match to DR5
dr5_sl = np.where((dr5.sightlines['PLATE'] == plate)
& (dr5.sightlines['FIB'] == fib))[0][0]
if (ml_abs['zabs'][idx] >= dr5.sightlines['Z_START'][dr5_sl]) & \
(ml_abs['zabs'][idx] <= dr5.sightlines['Z_END'][dr5_sl]):
pass
else:
fpos_in_dr5[idx] = False
print("Number of FP in DR5 analysis region = {:d}".format(
np.sum(fpos_in_dr5)))
# How many match to DR5 SLLS?
slls = all_dr5.NHI < 20.3
slls_coord = all_dr5.coord[slls]
slls_zabs = all_dr5.zabs[slls]
nslls = 0
for idx in np.where(fpos_in_dr5)[0]:
# Convoluted indexing..
mlid = ml_abs['ids'][idx]
# RA/DEC
ra = headers[mlid]['RA_GROUP']
dec = headers[mlid]['DEC_GROUP']
coord = SkyCoord(ra=ra, dec=dec, unit='deg')
# Match coord
mt = coord.separation(slls_coord) < 3 * u.arcsec
if np.any(mt):
# Match redshift
if np.min(np.abs(slls_zabs[mt] - ml_abs['zabs'][idx])) < 0.015:
nslls += 1
print("Number of FP that are SLLS in DR5 = {:d}".format(nslls))
low_NHI = ml_abs['NHI'][fpos_in_dr5] < 20.5
print("Number of FP that are NHI <= 20.5 = {:d}".format(np.sum(low_NHI)))
# Write out
fp_tbl = Table()
for key in ['ids', 'NHI', 'zabs', 'conf']:
fp_tbl[key] = ml_abs[key][fpos_in_dr5]
fp_tbl.write('test10k_false_pos.ascii',
format='ascii.fixed_width',
overwrite=True)
# Histogram
dr5_idx = np.where(fpos_in_dr5)
plt.clf()
ax = plt.gca()
ax.hist(ml_abs['conf'][dr5_idx])
plt.show()
############################################
if __name__ == '__main__':
args = parse_arguments()
if os.path.exists('mkdir -p QA/PNGs/'):
pass
else:
os.system('mkdir -p QA/PNGs/')
# ToDo -- fix ifs
if args.show:
gingashow = True
## Get tslits_dict either from flatfiles or load from a json file
if args.flatfiles is None:
jdict = ltu.loadjson('tilt_nires.json')
tslits_dict = jdict.copy()
for tkey in tslits_dict.keys():
tslits_dict[tkey] = np.array(tslits_dict[tkey])
else:
tslits_dict = get_tslits_nires(args.flatfiles, user_settings=par, gingashow=gingashow)
# Get Tilts from scienceB image
aImage = arcimage.ArcImage(spectrograph,
file_list=args.sciBfiles,
par=par['calibrations']['arcframe'])
msarc = aImage.process(bias_subtract='overscan',
trim=False)
pixlocn = pixels.gen_pixloc(aImage.stack.shape)
bpm = spectrograph.bpm(shape=msarc.shape, det=1)
# Extract spectrum at the center
def ARMLSD(argflag, spect, fitsdict, reuseMaster=False):
"""
Automatic Reduction and Modeling of Long Slit Data
Parameters
----------
argflag : dict
Arguments and flags used for reduction
spect : dict
Properties of the spectrograph.
fitsdict : dict
Contains relevant information from fits header files
reuseMaster : bool
If True, a master frame that will be used for another science frame
will not be regenerated after it is first made.
This setting comes with a price, and if a large number of science frames are
being generated, it may be more efficient to simply regenerate the master
calibrations on the fly.
Returns
-------
status : int
Status of the reduction procedure
0 = Successful execution
1 = ...
"""
status = 0
# Create a list of science exposure classes
sciexp = armbase.SetupScience(argflag, spect, fitsdict)
numsci = len(sciexp)
# Create a list of master calibration frames
masters = armasters.MasterFrames(spect['mosaic']['ndet'])
# Use Masters? Requires setup file
setup_file = argflag['out']['sorted']+'.setup'
try:
calib_dict = ltu.loadjson(setup_file)
except:
msgs.info("No setup file {:s} for MasterFrames".format(setup_file))
calib_dict = {}
else:
argflag['masters']['setup_file'] = setup_file
# Start reducing the data
for sc in range(numsci):
slf = sciexp[sc]
scidx = slf._idx_sci[0]
msgs.info("Reducing file {0:s}, target {1:s}".format(fitsdict['filename'][scidx], slf._target_name))
msgs.sciexp = slf # For QA writing on exit, if nothing else. Could write Masters too
# Loop on Detectors
for kk in xrange(slf._spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
slf.det = det
###############
# Get amplifier sections
arproc.get_ampsec_trimmed(slf, fitsdict, det, scidx)
# Setup
setup = arsort.calib_setup(slf, sc, det, fitsdict, calib_dict, write=False)
slf._argflag['masters']['setup'] = setup
###############
# Generate master bias frame
update = slf.MasterBias(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="bias")
###############
# Generate a bad pixel mask (should not repeat)
update = slf.BadPixelMask(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Generate a master arc frame
update = slf.MasterArc(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Determine the dispersion direction (and transpose if necessary)
slf.GetDispersionDirection(fitsdict, det, scidx)
if slf._bpix[det-1] is None: # Needs to be done here after nspec is set
slf.SetFrame(slf._bpix, np.zeros((slf._nspec[det-1], slf._nspat[det-1])), det)
'''
###############
# Estimate gain and readout noise for the amplifiers
msgs.work("Estimate Gain and Readout noise from the raw frames...")
update = slf.MasterRN(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="readnoise")
'''
###############
# Generate a master trace frame
update = slf.MasterTrace(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")
###############
# Generate an array that provides the physical pixel locations on the detector
slf.GetPixelLocations(det)
# Determine the edges of the spectrum (spatial)
if 'trace'+slf._argflag['masters']['setup'] not in slf._argflag['masters']['loaded']:
###############
# Determine the edges of the spectrum (spatial)
lordloc, rordloc, extord = artrace.trace_orders(slf, slf._mstrace[det-1], det, singleSlit=True, pcadesc="PCA trace of the slit edges")
slf.SetFrame(slf._lordloc, lordloc, det)
slf.SetFrame(slf._rordloc, rordloc, det)
# Convert physical trace into a pixel trace
msgs.info("Converting physical trace locations to nearest pixel")
pixcen = artrace.phys_to_pix(0.5*(slf._lordloc[det-1]+slf._rordloc[det-1]), slf._pixlocn[det-1], 1)
pixwid = (slf._rordloc[det-1]-slf._lordloc[det-1]).mean(0).astype(np.int)
lordpix = artrace.phys_to_pix(slf._lordloc[det-1], slf._pixlocn[det-1], 1)
rordpix = artrace.phys_to_pix(slf._rordloc[det-1], slf._pixlocn[det-1], 1)
slf.SetFrame(slf._pixcen, pixcen, det)
slf.SetFrame(slf._pixwid, pixwid, det)
slf.SetFrame(slf._lordpix, lordpix, det)
slf.SetFrame(slf._rordpix, rordpix, det)
# Save QA for slit traces
arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], slf._rordpix[det-1], extord, desc="Trace of the slit edges")
###############
# Prepare the pixel flat field frame
update = slf.MasterFlatField(fitsdict, det)
if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="pixflat")
###############
# Generate the 1D wavelength solution
update = slf.MasterWaveCalib(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="trace")
###############
# Derive the spectral tilt
if slf._tilts[det-1] is None:
if slf._argflag['masters']['use']:
mstilt_name = armasters.master_name(slf._argflag['run']['masterdir'],
'tilts', slf._argflag['masters']['setup'])
try:
tilts, head = arload.load_master(mstilt_name, frametype="tilts")
except IOError:
pass
else:
slf.SetFrame(slf._tilts, tilts, det)
slf._argflag['masters']['loaded'].append('tilts'+slf._argflag['masters']['setup'])
if 'tilts'+slf._argflag['masters']['setup'] not in slf._argflag['masters']['loaded']:
# First time tilts are derived for this arc frame --> derive the order tilts
tilts, satmask, outpar = artrace.model_tilt(slf, det, slf._msarc[det-1])
slf.SetFrame(slf._tilts, tilts, det)
slf.SetFrame(slf._satmask, satmask, det)
slf.SetFrame(slf._tiltpar, outpar, det)
###############
# Generate/load a master wave frame
update = slf.MasterWave(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="wave")
# Check if the user only wants to prepare the calibrations only
msgs.info("All calibration frames have been prepared")
if slf._argflag['run']['preponly']:
msgs.info("If you would like to continue with the reduction,"
+msgs.newline()+"disable the run+preponly command")
continue
# Write setup
setup = arsort.calib_setup(slf, sc, det, fitsdict, calib_dict, write=True)
# Write MasterFrames (currently per detector)
armasters.save_masters(slf, det, setup)
###############
# Load the science frame and from this generate a Poisson error frame
msgs.info("Loading science frame")
sciframe = arload.load_frames(slf, fitsdict, [scidx], det,
frametype='science',
msbias=slf._msbias[det-1],
transpose=slf._transpose)
sciframe = sciframe[:, :, 0]
# Extract
msgs.info("Processing science frame")
arproc.reduce_frame(slf, sciframe, scidx, fitsdict, det)
#continue
#msgs.error("UP TO HERE")
###############
# Perform a velocity correction
if (slf._argflag['reduce']['heliocorr'] == True) & False:
if slf._argflag['science']['load']['extracted'] == True:
msgs.warn("Heliocentric correction will not be applied if an extracted science frame exists, and is used")
msgs.work("Perform a full barycentric correction")
msgs.work("Include the facility to correct for gravitational redshifts and time delays (see Pulsar timing work)")
msgs.info("Performing a heliocentric correction")
# Load the header for the science frame
slf._waveids = arvcorr.helio_corr(slf, scidx[0])
else:
msgs.info("A heliocentric correction will not be performed")
###############
# Using model sky, calculate a flexure correction
# Close the QA for this object
slf._qa.close()
###############
# Flux
###############
# Standard star (is this a calibration, e.g. goes above?)
msgs.info("Processing standard star")
msgs.info("Assuming one star per detector mosaic")
msgs.info("Waited until last detector to process")
msgs.work("Need to check for existing sensfunc")
update = slf.MasterStandard(scidx, fitsdict)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, 0, ftype="standard")
#
msgs.work("Consider using archived sensitivity if not found")
msgs.info("Fluxing with {:s}".format(slf._sensfunc['std']['name']))
for kk in xrange(slf._spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
arflux.apply_sensfunc(slf, det, scidx, fitsdict)
# Write 1D spectra
arsave.save_1d_spectra(slf)
# Write 2D images for the Science Frame
arsave.save_2d_images(slf)
# Free up some memory by replacing the reduced ScienceExposure class
sciexp[sc] = None
return status
def __init__(self,
ispec,
guessfile=None,
parent=None,
zsys=None,
norm=None,
exten=None,
rsp_kwargs={},
unit_test=False,
screen_scale=1.,
**kwargs):
QMainWindow.__init__(self, parent)
"""
ispec = str, XSpectrum1D or tuple of arrays
Input spectrum or spectrum filename. If tuple then (wave,
fx), (wave, fx, sig) or (wave, fx, sig, co)
guessfile : str, optional
name of the .json file generated with igmguesses GUI in Pyigm (see https://github.com/pyigm/pyigm/blob/master/docs/igmguesses.rst)
if not None - overplot fitted line profiles from igmguesses
parent : Widget parent, optional
zsys : float, optional
intial redshift
exten : int, optional
extension for the spectrum in multi-extension FITS file
norm : bool, optional
True if the spectrum is normalized
screen_scale : float, optional
Scale the default sizes for the gui size
"""
#reload(ltgl)
#reload(ltgsp)
# INIT
#QtCore.pyqtRemoveInputHook()
#xdb.set_trace()
#QtCore.pyqtRestoreInputHook()
self.scale = screen_scale
# Needed to avoid crash in large spectral files
rcParams['agg.path.chunksize'] = 20000
rcParams[
'axes.formatter.useoffset'] = False # avoid scientific notation in axes tick labels
# Build a widget combining several others
self.main_widget = QWidget()
# Status bar
self.create_status_bar()
# Grab the pieces and tie together
self.pltline_widg = ltgl.PlotLinesWidget(status=self.statusBar,
init_z=zsys,
screen_scale=self.scale)
self.pltline_widg.setMaximumWidth(300 * self.scale)
## Abs sys
abs_sys = None
voigtsfit = None
if guessfile is not None:
# Load
ism = LineList('ISM')
igm_guess = ltu.loadjson(guessfile)
comps = []
for key in igm_guess['cmps'].keys():
comp = AbsComponent.from_dict(igm_guess['cmps'][key],
chk_vel=False,
linelist=ism)
comps.append(comp)
abs_sys = ltiu.build_systems_from_components(
comps, vsys=500. * u.km /
u.s) # ,chk_z=False) ### 100000.*u.km/u.s ok
### voigt fit - added
# Spectrum
spec, spec_fil = ltgu.read_spec(ispec,
exten=exten,
norm=norm,
rsp_kwargs=rsp_kwargs)
voigtsfit = np.asarray([0] * len(spec.wavelength))
alllines = []
for iabs_sys in abs_sys:
lines = iabs_sys.list_of_abslines()
alllines = alllines + lines
if len(alllines) > 0:
voigtsfit = lav.voigt_from_abslines(spec.wavelength,
alllines,
fwhm=3.).flux.value
if not norm:
voigtsfit = voigtsfit * spec.co
# Hook the spec widget to Plot Line
self.spec_widg = ltgsp.ExamineSpecWidget(ispec,
guessfile=guessfile,
voigtsfit=voigtsfit,
status=self.statusBar,
parent=self,
llist=self.pltline_widg.llist,
zsys=zsys,
norm=norm,
exten=exten,
abs_sys=abs_sys,
screen_scale=self.scale,
rsp_kwargs=rsp_kwargs,
**kwargs)
# Reset redshift from spec
if zsys is None:
if hasattr(self.spec_widg.spec, 'z'):
self.pltline_widg.setz(
str(self.spec_widg.spec.z[self.spec_widg.select]))
# Auto set line list if spec has proper object type
if hasattr(self.spec_widg.spec, 'stypes'):
if self.spec_widg.spec.stypes[
self.spec_widg.select].lower() == 'galaxy':
self.pltline_widg.llist = ltgu.set_llist(
'Galaxy', in_dict=self.pltline_widg.llist)
elif self.spec_widg.spec.stypes[
self.spec_widg.select].lower() == 'absorber':
self.pltline_widg.llist = ltgu.set_llist(
'Strong', in_dict=self.pltline_widg.llist)
self.pltline_widg.llist['Plot'] = True
idx = self.pltline_widg.lists.index(
self.pltline_widg.llist['List'])
self.pltline_widg.llist_widget.setCurrentRow(idx)
#
self.pltline_widg.spec_widg = self.spec_widg
# Multi spec
self.mspec_widg = ltgsp.MultiSpecWidget(self.spec_widg)
self.spec_widg.canvas.mpl_connect('button_press_event', self.on_click)
# Layout
# Extras
extras = QWidget()
extras.setMinimumWidth(180 * self.scale)
extras.setMaximumWidth(280 * self.scale)
vbox = QVBoxLayout()
qbtn = QPushButton(self)
qbtn.setText('Quit')
qbtn.clicked.connect(self.quit)
vbox.addWidget(self.pltline_widg)
vbox.addWidget(self.mspec_widg)
vbox.addWidget(qbtn)
extras.setLayout(vbox)
# Main window
hbox = QHBoxLayout()
hbox.addWidget(self.spec_widg)
hbox.addWidget(extras)
self.main_widget.setLayout(hbox)
# Point MainWindow
self.setCentralWidget(self.main_widget)
if unit_test:
self.quit()
def generate_boss_tables():
"""
Returns
-------
"""
# Load JSON file
dr12_json = resource_filename('dla_cnn',
'catalogs/boss_dr12/predictions_DR12.json')
dr12 = ltu.loadjson(dr12_json)
# Load Garnett Table 2 for BALs
tbl2_garnett_file = '/media/xavier/ExtraDrive2/Projects/ML_DLA_results/garnett16/ascii_catalog/table2.dat'
tbl2_garnett = Table.read(tbl2_garnett_file, format='cds')
tbl2_garnett_coords = SkyCoord(ra=tbl2_garnett['RAdeg'],
dec=tbl2_garnett['DEdeg'],
unit='deg')
# Parse into tables
s_plates = []
s_fibers = []
s_mjds = []
s_ra = []
s_dec = []
s_zem = []
a_zabs = []
a_NHI = []
a_sigNHI = []
a_conf = []
a_plates = []
a_fibers = []
a_mjds = []
a_ra = []
a_dec = []
a_zem = []
for sline in dr12:
# Plate/fiber
plate, mjd, fiber = [int(spl) for spl in sline['id'].split('-')]
s_plates.append(plate)
s_mjds.append(mjd)
s_fibers.append(fiber)
# RA/DEC/zem
s_ra.append(sline['ra'])
s_dec.append(sline['dec'])
s_zem.append(sline['z_qso'])
# DLAs/SLLS
for abs in sline['dlas'] + sline['subdlas']:
a_plates.append(plate)
a_mjds.append(mjd)
a_fibers.append(fiber)
# RA/DEC/zem
a_ra.append(sline['ra'])
a_dec.append(sline['dec'])
a_zem.append(sline['z_qso'])
# Absorber
a_zabs.append(abs['z_dla'])
a_NHI.append(abs['column_density'])
a_sigNHI.append(abs['std_column_density'])
a_conf.append(abs['dla_confidence'])
# Sightline tables
sline_tbl = Table()
sline_tbl['Plate'] = s_plates
sline_tbl['Fiber'] = s_fibers
sline_tbl['MJD'] = s_mjds
sline_tbl['RA'] = s_ra
sline_tbl['DEC'] = s_dec
sline_tbl['zem'] = s_zem
# Match and fill BAL flag
dr12_sline_coord = SkyCoord(ra=sline_tbl['RA'],
dec=sline_tbl['DEC'],
unit='deg')
sline_tbl['flg_BAL'] = -1
idx, d2d, d3d = match_coordinates_sky(dr12_sline_coord,
tbl2_garnett_coords,
nthneighbor=1)
in_garnett = d2d < 1 * u.arcsec # Check
sline_tbl['flg_BAL'][in_garnett] = tbl2_garnett['f_BAL'][idx[in_garnett]]
print("There were {:d} DR12 sightlines not in Garnett".format(
np.sum(~in_garnett)))
# Write
dr12_sline = resource_filename('dla_cnn',
'catalogs/boss_dr12/DR12_sightlines.fits')
sline_tbl.write(dr12_sline, overwrite=True)
print("Wrote {:s}".format(dr12_sline))
# DLA/SLLS table
abs_tbl = Table()
abs_tbl['Plate'] = a_plates
abs_tbl['Fiber'] = a_fibers
abs_tbl['MJD'] = a_mjds
abs_tbl['RA'] = a_ra
abs_tbl['DEC'] = a_dec
abs_tbl['zem'] = a_zem
#
abs_tbl['zabs'] = a_zabs
abs_tbl['NHI'] = a_NHI
abs_tbl['sigNHI'] = a_sigNHI
abs_tbl['conf'] = a_conf
# BAL
dr12_abs_coord = SkyCoord(ra=abs_tbl['RA'], dec=abs_tbl['DEC'], unit='deg')
idx, d2d, d3d = match_coordinates_sky(dr12_abs_coord,
tbl2_garnett_coords,
nthneighbor=1)
in_garnett = d2d < 1 * u.arcsec # Check
abs_tbl['flg_BAL'] = -1
abs_tbl['flg_BAL'][in_garnett] = tbl2_garnett['f_BAL'][idx[in_garnett]]
abs_tbl['SNR'] = 0.
abs_tbl['SNR'][in_garnett] = tbl2_garnett['SNRSpec'][idx[in_garnett]]
print("There were {:d} DR12 absorbers not covered by Garnett".format(
np.sum(~in_garnett)))
dr12_abs = resource_filename('dla_cnn',
'catalogs/boss_dr12/DR12_DLA_SLLS.fits')
abs_tbl.write(dr12_abs, overwrite=True)
print("Wrote {:s}".format(dr12_abs))
# Garnett
ml_path = os.getenv('PROJECT_ML')
g16_dlas = Table.read(ml_path + '/garnett16/ascii_catalog/table3.dat',
format='cds')
tbl3_garnett_coords = SkyCoord(ra=g16_dlas['RAdeg'],
dec=g16_dlas['DEdeg'],
unit='deg')
idx, d2d, d3d = match_coordinates_sky(tbl3_garnett_coords,
tbl2_garnett_coords,
nthneighbor=1)
in_garnett = d2d < 1 * u.arcsec # Check
g16_dlas['flg_BAL'] = -1
g16_dlas['flg_BAL'][in_garnett] = tbl2_garnett['f_BAL'][idx[in_garnett]]
g16_dlas['SNR'] = 0.
g16_dlas['SNR'][in_garnett] = tbl2_garnett['SNRSpec'][idx[in_garnett]]
g16_outfile = resource_filename(
'dla_cnn', 'catalogs/boss_dr12/DR12_DLA_garnett16.fits')
g16_dlas.write(g16_outfile, overwrite=True)
print("Wrote {:s}".format(g16_outfile))
import numpy as np
from astropy.table import vstack
from linetools import utils as ltu
from linetools.spectra.xspectrum1d import XSpectrum1D
from pypeit.core.wavecal import autoid
from pypeit.core.wavecal import waveio
from pypeit.core import arc
from pypeit.spectrographs import gemini_gmos
# Load the spectra
chip = 2
if chip == 1:
jdict = ltu.loadjson('GMOS_R400_blue.json.gz')
outroot = 'GMOS_R400_blue_'
elif chip == 2:
jdict = ltu.loadjson('GMOS_R400_chip2.json.gz')
outroot = 'GMOS_R400_chip2_'
spectrograph = gemini_gmos.GeminiGMOSNE2VSpectrograph()
arcparam = {}
spectrograph.setup_arcparam(arcparam, disperser='R400')
arcparam['n_first'] = 2
arcparam['n_final'] = 3
arcparam['func'] = 'legendre'
arcparam['nsig_rej'] = 2.
arcparam['nsig_rej_final'] = 3.
arcparam['disp'] = 0.67 * 2.
arcparam['match_toler'] = 3.
hdu_sedg = fits.open(user +
'/Dropbox/Cowie_2002-02-17/Flats/SEdgECH75_1x1.fits')
data = hdu_sedg[0].data
slit_left = data[0, :, :].T
slit_righ = data[1, :, :].T
plate_scale = 0.149
hdu_std = fits.open(
user +
'/Dropbox/Cowie_2002-02-17/Extract/Obj_ES.20020217.20037.fits.gz')
std_trace = hdu_std[1].data['trace'][:, 0:slit_left.shape[0]].T
std_trace[:, -1] = std_trace[:, -2] + (np.median(std_trace[:200, -1]) -
np.median(std_trace[:200, -2]))
#The structure is exten = 1, and the xpos,ypos are the standard trace
elif spectro == 'NIRES':
from linetools import utils as ltu
jdict = ltu.loadjson(user + '/Dropbox/hires_fndobj/tilt_nires.json')
slit_left = np.array(jdict['lcen'])
slit_righ = np.array(jdict['rcen'])
hdu = fits.open(
user +
'Dropbox/hires_fndobj/spec2d_J1724+1901_NIRES_2018Jun04T130207.856.fits'
)
objminsky = hdu[1].data - hdu[3].data
ivar = hdu[2].data
mask = (ivar > 0)
plate_scale = 0.123
std_trace = None
elif spectro == 'GNIRS':
from scipy.io import readsav
#hdu = fits.open(user + '/Dropbox/hires_fndobj/sci-N20170331S0216-219.fits')
#hdu = fits.open(user + '/Dropbox/hires_fndobj/GNIRS/J021514.76+004223.8/Science/J021514.76+004223.8_1/sci-N20170927S0294-297.fits')
ivar = utils.calc_ivar(var)
mask = (var > 0.0)
skyimg = hdu[2].data
objminsky = sciimg - skyimg
hdu_sedg = fits.open(user + '/Dropbox/Cowie_2002-02-17/Flats/SEdgECH75_1x1.fits')
data = hdu_sedg[0].data
slit_left = data[0,:,:].T
slit_righ = data[1,:,:].T
plate_scale = 0.149
hdu_std =fits.open(user + '/Dropbox/Cowie_2002-02-17/Extract/Obj_ES.20020217.20037.fits.gz')
std_trace = hdu_std[1].data['trace'][:,0:slit_left.shape[0]].T
std_trace[:,-1] = std_trace[:,-2]+ (np.median(std_trace[:200,-1])-np.median(std_trace[:200,-2]))
#The structure is exten = 1, and the xpos,ypos are the standard trace
elif spectro == 'NIRES':
from linetools import utils as ltu
jdict = ltu.loadjson(user + '/Dropbox/hires_fndobj/tilt_nires.json')
slit_left = np.array(jdict['lcen'])
slit_righ = np.array(jdict['rcen'])
hdu = fits.open(user + 'Dropbox/hires_fndobj/spec2d_J1724+1901_NIRES_2018Jun04T130207.856.fits')
objminsky = hdu[1].data - hdu[3].data
ivar = hdu[2].data
mask = (ivar>0)
plate_scale = 0.123
std_trace = None
elif spectro == 'GNIRS':
from scipy.io import readsav
#hdu = fits.open(user + '/Dropbox/hires_fndobj/sci-N20170331S0216-219.fits')
#hdu = fits.open(user + '/Dropbox/hires_fndobj/GNIRS/J021514.76+004223.8/Science/J021514.76+004223.8_1/sci-N20170927S0294-297.fits')
#hdu = fits.open(user + 'Dropbox/hires_fndobj/GNIRS/J005424.45+004750.2/Science/J005424.45+004750.2_7/sci-N20171021S0264-267.fits')
hdu = fits.open(user + 'Dropbox/hires_fndobj/GNIRS/J002407.02-001237.2/Science/J002407.02-001237.2_5/sci-N20171006S0236-239.fits')
obj = hdu[0].data
def load_sys_files(inp,
type,
ref=None,
sys_path=False,
build_abs_sys=False,
**kwargs):
""" Load up a set of SYS files from the hard-drive (JSON files)
Parameters
----------
inp : str
Name of JSON tarball or if sys_path=True then the path to a folder of JSON files
type : str
type of IGMSystem, e.g. LLS
ref : str, optional
Reference label
sys_path : str, optional
indicates that inp is a path to a set of JSON SYS files
otherwise, inp should be the filename of a tarball of JSON files
build_abs_sys : bool, optional
Build a list of AbsSystem's? Can always be instantiated later
**kwargs :
Passed to system
Returns
-------
survey : IGMSurvey
"""
import tarfile
#
survey = class_by_type(type)(ref=ref)
system = pyasu.class_by_type(type)
if sys_path:
pdb.set_trace() # THIS NEEDS TO BE UPDATED AS WAS DONE FOR THE TARBALL
# Individual files
files = glob.glob(inp + '*.json')
files.sort()
for ifile in files:
tdict = ltu.loadjson(ifile)
abssys = system.from_dict(tdict, linelist=llist)
survey._abs_sys.append(abssys)
else: # tarball
print('Loading systems from {:s}'.format(inp))
tar = tarfile.open(inp, 'r:gz')
for member in tar.getmembers():
if '.' not in member.name:
print('Skipping a likely folder: {:s}'.format(member.name))
continue
# Extract
f = tar.extractfile(member)
f = f.read()
f = f.decode('utf-8')
tdict = json.loads(f)
# Add keys (for backwards compatability)
if ('NHI' in tdict.keys()) and ('flag_NHI' not in tdict.keys()):
tdict['flag_NHI'] = 1
# Add to list of dicts
survey._dict[tdict['Name']] = tdict
tar.close()
# Mask
survey.init_mask()
# Set coordinates
ras = [survey._dict[key]['RA'] for key in survey._dict.keys()]
decs = [survey._dict[key]['DEC'] for key in survey._dict.keys()]
survey.coords = SkyCoord(ra=ras, dec=decs, unit='deg')
# Build AbsSystem objects?
if build_abs_sys:
survey.build_all_abs_sys(linelist=llist)
# Generate the data table
print("Building the data Table from the internal dict")
survey.data_from_dict()
# Return
return survey
def mk_db(dbname, tree, outfil, iztbl, version='v00', id_key='PRIV_ID',
publisher='Unknown', **kwargs):
""" Generate the DB
Parameters
----------
dbname : str
Name for the database
tree : str
Path to top level of the tree of FITS files
Typically, each branch in the tree corresponds to a single instrument
outfil : str
Output file name for the hdf5 file
iztbl : Table or str
If Table, see meta() docs for details on its format
If str, it must be 'igmspec' and the user must have that DB downloaded
version : str, optional
Version code
Returns
-------
"""
from specdb import defs
# ztbl
if isinstance(iztbl, str):
if iztbl == 'igmspec':
from specdb.specdb import IgmSpec
igmsp = IgmSpec()
ztbl = Table(igmsp.idb.hdf['quasars'][...])
elif isinstance(iztbl, Table):
ztbl = iztbl
else:
raise IOError("Bad type for ztbl")
# Find the branches
branches = glob.glob(tree+'/*')
branches.sort()
# HDF5 file
hdf = h5py.File(outfil,'w')
# Defs
zpri = defs.z_priority()
gdict = {}
# Main DB Table
maindb, tkeys = spbu.start_maindb(id_key)
# MAIN LOOP
for ss,branch in enumerate(branches):
# Skip files
if not os.path.isdir(branch):
continue
print('Working on branch: {:s}'.format(branch))
# Files
fits_files, out_tup = grab_files(branch)
meta_file, mtbl_file, ssa_file = out_tup
# Meta
maxpix, phead, mdict, stype = 10000, None, None, 'QSO'
if meta_file is not None:
# Load
meta_dict = ltu.loadjson(meta_file)
# Maxpix
if 'maxpix' in meta_dict.keys():
maxpix = meta_dict['maxpix']
# STYPE
if 'stype' in meta_dict.keys():
stype = meta_dict['stype']
# Parse header
if 'parse_head' in meta_dict.keys():
phead = meta_dict['parse_head']
if 'meta_dict' in meta_dict.keys():
mdict = meta_dict['meta_dict']
full_meta = mk_meta(fits_files, ztbl, mtbl_file=mtbl_file,
parse_head=phead, mdict=mdict, **kwargs)
# Update group dict
group_name = branch.split('/')[-1]
flag_g = spbu.add_to_group_dict(group_name, gdict)
# IDs
maindb = add_ids(maindb, full_meta, flag_g, tkeys, 'PRIV_ID', first=(flag_g==1))
# Ingest
ingest_spectra(hdf, group_name, full_meta, max_npix=maxpix, **kwargs)
# SSA
if ssa_file is not None:
user_ssa = ltu.loadjson(ssa_file)
ssa_dict = default_fields(user_ssa['Title'], flux=user_ssa['flux'], fxcalib=user_ssa['fxcalib'])
hdf[group_name]['meta'].attrs['SSA'] = json.dumps(ltu.jsonify(ssa_dict))
# Check stacking
if not spbu.chk_vstack(hdf):
print("Meta data will not stack using specdb.utils.clean_vstack")
print("Proceed to write at your own risk..")
pdb.set_trace()
# Write
write_hdf(hdf, str(dbname), maindb, zpri, gdict, version,
Publisher=publisher)
print("Wrote {:s} DB file".format(outfil))
def ARMLSD(argflag, spect, fitsdict, reuseMaster=False):
"""
Automatic Reduction and Modeling of Long Slit Data
Parameters
----------
argflag : dict
Arguments and flags used for reduction
spect : dict
Properties of the spectrograph.
fitsdict : dict
Contains relevant information from fits header files
reuseMaster : bool
If True, a master frame that will be used for another science frame
will not be regenerated after it is first made.
This setting comes with a price, and if a large number of science frames are
being generated, it may be more efficient to simply regenerate the master
calibrations on the fly.
Returns
-------
status : int
Status of the reduction procedure
0 = Successful execution
1 = ...
"""
status = 0
# Create a list of science exposure classes
sciexp = armbase.SetupScience(argflag, spect, fitsdict)
numsci = len(sciexp)
# Create a list of master calibration frames
masters = armasters.MasterFrames(spect['mosaic']['ndet'])
# Use Masters? Requires setup file
setup_file = argflag['out']['sorted'] + '.setup'
try:
calib_dict = ltu.loadjson(setup_file)
except:
msgs.info("No setup file {:s} for MasterFrames".format(setup_file))
calib_dict = {}
else:
argflag['masters']['setup_file'] = setup_file
# Start reducing the data
for sc in range(numsci):
slf = sciexp[sc]
scidx = slf._idx_sci[0]
msgs.info("Reducing file {0:s}, target {1:s}".format(
fitsdict['filename'][scidx], slf._target_name))
msgs.sciexp = slf # For QA writing on exit, if nothing else. Could write Masters too
# Loop on Detectors
for kk in xrange(slf._spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
slf.det = det
###############
# Get amplifier sections
arproc.get_ampsec_trimmed(slf, fitsdict, det, scidx)
# Setup
setup = arsort.calib_setup(slf,
sc,
det,
fitsdict,
calib_dict,
write=False)
slf._argflag['masters']['setup'] = setup
###############
# Generate master bias frame
update = slf.MasterBias(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="bias")
###############
# Generate a bad pixel mask (should not repeat)
update = slf.BadPixelMask(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Generate a master arc frame
update = slf.MasterArc(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Determine the dispersion direction (and transpose if necessary)
slf.GetDispersionDirection(fitsdict, det, scidx)
if slf._bpix[
det -
1] is None: # Needs to be done here after nspec is set
slf.SetFrame(
slf._bpix,
np.zeros((slf._nspec[det - 1], slf._nspat[det - 1])), det)
'''
###############
# Estimate gain and readout noise for the amplifiers
msgs.work("Estimate Gain and Readout noise from the raw frames...")
update = slf.MasterRN(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="readnoise")
'''
###############
# Generate a master trace frame
update = slf.MasterTrace(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp,
sc,
det,
ftype="flat",
chktype="trace")
###############
# Generate an array that provides the physical pixel locations on the detector
slf.GetPixelLocations(det)
# Determine the edges of the spectrum (spatial)
if 'trace' + slf._argflag['masters']['setup'] not in slf._argflag[
'masters']['loaded']:
###############
# Determine the edges of the spectrum (spatial)
lordloc, rordloc, extord = artrace.trace_orders(
slf,
slf._mstrace[det - 1],
det,
singleSlit=True,
pcadesc="PCA trace of the slit edges")
slf.SetFrame(slf._lordloc, lordloc, det)
slf.SetFrame(slf._rordloc, rordloc, det)
# Convert physical trace into a pixel trace
msgs.info(
"Converting physical trace locations to nearest pixel")
pixcen = artrace.phys_to_pix(
0.5 * (slf._lordloc[det - 1] + slf._rordloc[det - 1]),
slf._pixlocn[det - 1], 1)
pixwid = (slf._rordloc[det - 1] -
slf._lordloc[det - 1]).mean(0).astype(np.int)
lordpix = artrace.phys_to_pix(slf._lordloc[det - 1],
slf._pixlocn[det - 1], 1)
rordpix = artrace.phys_to_pix(slf._rordloc[det - 1],
slf._pixlocn[det - 1], 1)
slf.SetFrame(slf._pixcen, pixcen, det)
slf.SetFrame(slf._pixwid, pixwid, det)
slf.SetFrame(slf._lordpix, lordpix, det)
slf.SetFrame(slf._rordpix, rordpix, det)
# Save QA for slit traces
arqa.slit_trace_qa(slf,
slf._mstrace[det - 1],
slf._lordpix[det - 1],
slf._rordpix[det - 1],
extord,
desc="Trace of the slit edges")
###############
# Prepare the pixel flat field frame
update = slf.MasterFlatField(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp,
sc,
det,
ftype="flat",
chktype="pixflat")
###############
# Generate the 1D wavelength solution
update = slf.MasterWaveCalib(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp,
sc,
det,
ftype="arc",
chktype="trace")
###############
# Derive the spectral tilt
if slf._tilts[det - 1] is None:
if slf._argflag['masters']['use']:
mstilt_name = armasters.master_name(
slf._argflag['run']['masterdir'], 'tilts',
slf._argflag['masters']['setup'])
try:
tilts, head = arload.load_master(mstilt_name,
frametype="tilts")
except IOError:
pass
else:
slf.SetFrame(slf._tilts, tilts, det)
slf._argflag['masters']['loaded'].append(
'tilts' + slf._argflag['masters']['setup'])
if 'tilts' + slf._argflag['masters'][
'setup'] not in slf._argflag['masters']['loaded']:
# First time tilts are derived for this arc frame --> derive the order tilts
tilts, satmask, outpar = artrace.model_tilt(
slf, det, slf._msarc[det - 1])
slf.SetFrame(slf._tilts, tilts, det)
slf.SetFrame(slf._satmask, satmask, det)
slf.SetFrame(slf._tiltpar, outpar, det)
###############
# Generate/load a master wave frame
update = slf.MasterWave(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp,
sc,
det,
ftype="arc",
chktype="wave")
# Check if the user only wants to prepare the calibrations only
msgs.info("All calibration frames have been prepared")
if slf._argflag['run']['preponly']:
msgs.info("If you would like to continue with the reduction," +
msgs.newline() + "disable the run+preponly command")
continue
# Write setup
setup = arsort.calib_setup(slf,
sc,
det,
fitsdict,
calib_dict,
write=True)
# Write MasterFrames (currently per detector)
armasters.save_masters(slf, det, setup)
###############
# Load the science frame and from this generate a Poisson error frame
msgs.info("Loading science frame")
sciframe = arload.load_frames(slf,
fitsdict, [scidx],
det,
frametype='science',
msbias=slf._msbias[det - 1],
transpose=slf._transpose)
sciframe = sciframe[:, :, 0]
# Extract
msgs.info("Processing science frame")
arproc.reduce_frame(slf, sciframe, scidx, fitsdict, det)
#continue
#msgs.error("UP TO HERE")
###############
# Perform a velocity correction
if (slf._argflag['reduce']['heliocorr'] == True) & False:
if slf._argflag['science']['load']['extracted'] == True:
msgs.warn(
"Heliocentric correction will not be applied if an extracted science frame exists, and is used"
)
msgs.work("Perform a full barycentric correction")
msgs.work(
"Include the facility to correct for gravitational redshifts and time delays (see Pulsar timing work)"
)
msgs.info("Performing a heliocentric correction")
# Load the header for the science frame
slf._waveids = arvcorr.helio_corr(slf, scidx[0])
else:
msgs.info("A heliocentric correction will not be performed")
###############
# Using model sky, calculate a flexure correction
# Close the QA for this object
slf._qa.close()
###############
# Flux
###############
# Standard star (is this a calibration, e.g. goes above?)
msgs.info("Processing standard star")
msgs.info("Assuming one star per detector mosaic")
msgs.info("Waited until last detector to process")
msgs.work("Need to check for existing sensfunc")
update = slf.MasterStandard(scidx, fitsdict)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, 0, ftype="standard")
#
msgs.work("Consider using archived sensitivity if not found")
msgs.info("Fluxing with {:s}".format(slf._sensfunc['std']['name']))
for kk in xrange(slf._spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
arflux.apply_sensfunc(slf, det, scidx, fitsdict)
# Write 1D spectra
arsave.save_1d_spectra(slf)
# Write 2D images for the Science Frame
arsave.save_2d_images(slf)
# Free up some memory by replacing the reduced ScienceExposure class
sciexp[sc] = None
return status
def main(args=None):
pargs = parser(options=args)
import sys
import os
from linetools import utils as ltu
from vetrr.vet_redrock import VetRedRockGui
from PyQt5.QtWidgets import QApplication
from collections import OrderedDict
import yaml
import glob
if pargs.initials:
initials = pargs.initials
else:
initials = None
print(initials)
# get the local red rock file (input to vetrr)
# first check if its there, if not look above
if len(glob.glob("J*_rr.fits")) != 0:
infile = glob.glob("J*_rr.fits")[0]
elif glob.glob("J*_rr.fits") == 0:
infile = glob.glob("../J*_rr.fits")[0]
else:
import pdb
pdb.set_trace()
# Get the coadd_file (yaml)
if len(glob.glob("J*coadd.yaml")) != 0:
coadd = glob.glob("J*coadd.yaml")[0]
else:
import pdb
pdb.set_trace()
# check to see if there are any previous vetrr.json files
outguess = glob.glob("J*vetrr*.json")
# This is the case when no initials were supplied but no
# previous json file either.
if len(outguess) == 0:
if initials is not None:
outfile = infile[:-7] + "vetrr_" + initials + ".json"
else:
print("ERROR: No existing output. Need an outfile and initials")
outfile = None
elif initials is not None:
print("Createing a new output file")
outfile = infile[:-7] + "vetrr_" + initials + ".json"
else:
outfile = outguess[0]
print("******************************************************")
print("Auto found these files:")
print("INFILE:", infile)
print("OUTFILE:", outfile)
print("COADD:", coadd)
print("****************************************************** \n")
# Load outfile if it exists
if os.path.isfile(outfile):
print("******************************************************")
print("WARNING: Loading previous file and will over-write it!")
print("******************************************************")
zdict = ltu.loadjson(outfile)
zdict = OrderedDict(zdict)
else:
zdict = None
# YAML coadd file?
if coadd is not None:
# Load the input file
with open(coadd, 'r') as in_file:
coadd_dict = yaml.load(in_file)
app = QApplication(sys.argv)
gui = VetRedRockGui(infile,
outfile=outfile,
zdict=zdict,
coadd_dict=coadd_dict)
gui.show()
app.exec_()
# ESI
hdu = fits.open('/Users/feige/Dropbox/Cowie_2002-02-17/Final/fringe_ES.20020217.35453.fits.gz')
sciimg = hdu[0].data
var = hdu[1].data
ivar = utils.calc_ivar(var)
mask = (var > 0.0)
skyimg = hdu[2].data
objminsky = sciimg - skyimg
hdu_sedg = fits.open('/Users/feige/Dropbox/Cowie_2002-02-17/Flats/SEdgECH75_1x1.fits')
data = hdu_sedg[0].data
slit_left = data[0,:,:].T
slit_righ = data[1,:,:].T
plate_scale = 0.149
elif spectro == 'NIRES':
from linetools import utils as ltu
jdict = ltu.loadjson('/Users/feige/Dropbox/hires_fndobj/tilt_nires.json')
slit_left = np.array(jdict['lcen'])
slit_righ = np.array(jdict['rcen'])
hdu = fits.open('/Users/feige/Dropbox/hires_fndobj/spec2d_J1724+1901_NIRES_2018Jun04T130207.856.fits')
objminsky = hdu[1].data - hdu[3].data
ivar = hdu[2].data
mask = (ivar>0)
plate_scale = 0.123
elif spectro == 'GNIRS':
from scipy.io import readsav
#hdu = fits.open('/Users/feige/Dropbox/hires_fndobj/sci-N20170331S0216-219.fits')
hdu = fits.open('/Users/feige/Dropbox/hires_fndobj/GNIRS/J021514.76+004223.8/Science/J021514.76+004223.8_1/sci-N20170927S0294-297.fits')
hdu = fits.open('/Users/feige/Dropbox/hires_fndobj/GNIRS/J005424.45+004750.2/Science/J005424.45+004750.2_7/sci-N20171021S0264-267.fits')
hdu = fits.open('/Users/feige/Dropbox/hires_fndobj/GNIRS/J002407.02-001237.2/Science/J002407.02-001237.2_5/sci-N20171006S0236-239.fits')
obj = hdu[0].data
#objminsky = obj - hdu[1].data
def from_igmguesses(cls,
igmgfile,
name=None,
radec=None,
zem=None,
**kwargs):
""" Instantiate from a JSON file from IGMGuesses
Parameters
----------
igmgfile : str
Filename
name : str, optional
Name of the IGMSightline
radec : RA/DEC input, optional
See ltu.radec_to_coord for options on format
If given, it will overwrite the RA/DEC in IGMGuesses file (if any)
zem : float, optional
Emission redshift of sightline
If given, it will overwrite the zem in IGMGuesses file (if any)
Returns
-------
"""
from linetools.isgm import abscomponent
# Read
jdict = ltu.loadjson(igmgfile) # cmps, specfile, meta
# Add in additional keys
# Coords
if radec is not None:
coord = ltu.radec_to_coord(radec)
else:
coord = SkyCoord(jdict['meta']['RA'],
jdict['meta']['DEC'],
unit='deg')
jdict['RA'] = coord.icrs.ra.deg
jdict['DEC'] = coord.icrs.dec.deg
# zem
if zem is not None:
jdict['zem'] = zem
else:
jdict['zem'] = jdict['meta']['zem']
if jdict[
'zem'] == 0.: # conforming IGMGuesses current rule zem = 0. means zem not set
jdict['zem'] = None
# Name
if name is None:
if jdict['zem'] is None:
zem_name = 'z-unknown'
else:
zem_name = 'z{:0.3f}'.format(jdict['zem'])
name = 'J{:s}{:s}_{:s}'.format(
coord.icrs.ra.to_string(unit=u.hour, sep='', pad=True)[0:4],
coord.icrs.dec.to_string(sep='', pad=True,
alwayssign=True)[0:5], zem_name)
jdict['name'] = name
# Components
jdict['components'] = jdict.pop('cmps')
kwargs['use_coord'] = True
slf = cls.from_dict(jdict, **kwargs)
# Separate IGMGuesses attributes from AbsComponent
acomp_keys = list(abscomponent.init_attrib.keys())
for comp in slf._components:
comp.igmg_attrib = {}
akeys = list(comp.attrib.keys())
for key in akeys:
if key in acomp_keys:
pass
else:
comp.igmg_attrib[key] = comp.attrib.pop(key)
# Add other IGMGuesses specific attributes
comp.igmg_attrib['top_level'] = {}
for key in [
'Nfit', 'bfit', 'zfit', 'mask_abslines', 'wrest', 'vlim'
]: # I added vlim because these aren't always consistent. Must be something bad in igmguesses
if key in jdict['components'][comp.name].keys():
comp.igmg_attrib['top_level'][key] = jdict['components'][
comp.name][key]
# Slurp a few other IGMGuesses things
slf.igmg_dict = {}
for key in ['spec_file', 'meta', 'fwhm']:
slf.igmg_dict[key] = jdict[key]
# Return
return slf
def load_ml_file(pred_file):
""" Load the search results from the CNN into a DLASurvey object
Parameters
----------
pred_file
Returns
-------
ml_llssurvey: LLSSurvey
ml_dlasusrvey: DLASurvey
"""
print("Loading {:s}. Please be patient..".format(pred_file))
# Read
ml_results = ltu.loadjson(pred_file)
use_platef = False
if 'plate' in ml_results[0].keys():
use_platef = True
else:
if 'id' in ml_results[0].keys():
use_id = True
# Init
idict = dict(ra=[], dec=[], plate=[], fiber=[])
if use_platef:
for key in ['plate', 'fiber', 'mjd']:
idict[key] = []
dlasystems = []
llssystems = []
# Generate coords to speed things up
for obj in ml_results:
for key in ['ra', 'dec']:
idict[key].append(obj[key])
ml_coords = SkyCoord(ra=idict['ra'], dec=idict['dec'], unit='deg')
ra_names = ml_coords.icrs.ra.to_string(unit=u.hour, sep='', pad=True)
dec_names = ml_coords.icrs.dec.to_string(sep='', pad=True, alwayssign=True)
vlim = [-500., 500.] * u.km / u.s
dcoord = SkyCoord(ra=0., dec=0., unit='deg')
# Loop on list
didx, lidx = [], []
print("Looping on sightlines..")
for tt, obj in enumerate(ml_results):
#if (tt % 100) == 0:
# print('tt: {:d}'.format(tt))
# Sightline
if use_id:
plate, fiber = [int(spl) for spl in obj['id'].split('-')]
idict['plate'].append(plate)
idict['fiber'].append(fiber)
# Systems
for ss, syskey in enumerate(['dlas', 'subdlas']):
for idla in obj[syskey]:
name = 'J{:s}{:s}_z{:.3f}'.format(ra_names[tt], dec_names[tt],
idla['z_dla'])
if ss == 0:
isys = DLASystem(dcoord,
idla['z_dla'],
vlim,
NHI=idla['column_density'],
zem=obj['z_qso'],
name=name)
else:
isys = LLSSystem(dcoord,
idla['z_dla'],
vlim,
NHI=idla['column_density'],
zem=obj['z_qso'],
name=name)
isys.confidence = idla['dla_confidence']
isys.s2n = idla['s2n']
if use_platef:
isys.plate = obj['plate']
isys.fiber = obj['fiber']
elif use_id:
isys.plate = plate
isys.fiber = fiber
# Save
if ss == 0:
didx.append(tt)
dlasystems.append(isys)
else:
lidx.append(tt)
llssystems.append(isys)
# Generate sightline tables
sightlines = Table()
sightlines['RA'] = idict['ra']
sightlines['DEC'] = idict['dec']
sightlines['PLATE'] = idict['plate']
sightlines['FIBERID'] = idict['fiber']
# Surveys
ml_llssurvey = LLSSurvey()
ml_llssurvey.sightlines = sightlines.copy()
ml_llssurvey._abs_sys = llssystems
ml_llssurvey.coords = ml_coords[np.array(lidx)]
ml_dlasurvey = DLASurvey()
ml_dlasurvey.sightlines = sightlines.copy()
ml_dlasurvey._abs_sys = dlasystems
ml_dlasurvey.coords = ml_coords[np.array(didx)]
# Return
return ml_llssurvey, ml_dlasurvey
)
sciimg = hdu[0].data
var = hdu[1].data
ivar = utils.calc_ivar(var)
mask = (var > 0.0)
skyimg = hdu[2].data
objminsky = sciimg - skyimg
hdu_sedg = fits.open(
'/Users/feige/Dropbox/Cowie_2002-02-17/Flats/SEdgECH75_1x1.fits')
data = hdu_sedg[0].data
slit_left = data[0, :, :].T
slit_righ = data[1, :, :].T
plate_scale = 0.149
elif spectro == 'NIRES':
from linetools import utils as ltu
jdict = ltu.loadjson('/Users/feige/Dropbox/hires_fndobj/tilt_nires.json')
slit_left = np.array(jdict['lcen'])
slit_righ = np.array(jdict['rcen'])
hdu = fits.open(
'/Users/feige/Dropbox/hires_fndobj/spec2d_J1724+1901_NIRES_2018Jun04T130207.856.fits'
)
objminsky = hdu[1].data - hdu[3].data
ivar = hdu[2].data
mask = (ivar > 0)
plate_scale = 0.123
elif spectro == 'GNIRS':
from scipy.io import readsav
#hdu = fits.open('/Users/feige/Dropbox/hires_fndobj/sci-N20170331S0216-219.fits')
hdu = fits.open(
'/Users/feige/Dropbox/hires_fndobj/GNIRS/J021514.76+004223.8/Science/J021514.76+004223.8_1/sci-N20170927S0294-297.fits'
)
def json_to_sdss_dlasurvey(json_file, sdss_survey, add_pf=True, debug=False):
""" Convert JSON output file to a DLASurvey object
Assumes SDSS bookkeeping for sightlines (i.e. PLATE, FIBER)
Parameters
----------
json_file : str
Full path to the JSON results file
sdss_survey : DLASurvey
SDSS survey, usually human (e.g. JXP for DR5)
add_pf : bool, optional
Add plate/fiber to DLAs in sdss_survey
Returns
-------
ml_survey : LLSSurvey
Survey object for the LLS
"""
print("Loading SDSS Survey from JSON file {:s}".format(json_file))
# imports
from pyigm.abssys.dla import DLASystem
from pyigm.abssys.lls import LLSSystem
# Fiber key
for fkey in ['FIBER', 'FIBER_ID', 'FIB']:
if fkey in sdss_survey.sightlines.keys():
break
# Read
ml_results = ltu.loadjson(json_file)
use_platef = False
if 'plate' in ml_results[0].keys():
use_platef = True
else:
if 'id' in ml_results[0].keys():
use_id = True
# Init
#idict = dict(plate=[], fiber=[], classification_confidence=[], # FOR v2
# classification=[], ra=[], dec=[])
idict = dict(ra=[], dec=[])
if use_platef:
for key in ['plate', 'fiber', 'mjd']:
idict[key] = []
ml_tbl = Table()
ml_survey = LLSSurvey()
systems = []
in_ml = np.array([False]*len(sdss_survey.sightlines))
# Loop
for obj in ml_results:
# Sightline
for key in idict.keys():
idict[key].append(obj[key])
# DLAs
#if debug:
# if (obj['plate'] == 1366) & (obj['fiber'] == 614):
# sv_coord = SkyCoord(ra=obj['ra'], dec=obj['dec'], unit='deg')
# print("GOT A MATCH IN RESULTS FILE")
for idla in obj['dlas']:
"""
dla = DLASystem((sdss_survey.sightlines['RA'][mt[0]],
sdss_survey.sightlines['DEC'][mt[0]]),
idla['spectrum']/(1215.6701)-1., None,
idla['column_density'])
"""
if idla['z_dla'] < 1.8:
continue
isys = LLSSystem((obj['ra'],obj['dec']),
idla['z_dla'], None, NHI=idla['column_density'], zem=obj['z_qso'])
isys.confidence = idla['dla_confidence']
if use_platef:
isys.plate = obj['plate']
isys.fiber = obj['fiber']
elif use_id:
plate, fiber = [int(spl) for spl in obj['id'].split('-')]
isys.plate = plate
isys.fiber = fiber
# Save
systems.append(isys)
# Connect to sightlines
ml_coord = SkyCoord(ra=idict['ra'], dec=idict['dec'], unit='deg')
s_coord = SkyCoord(ra=sdss_survey.sightlines['RA'], dec=sdss_survey.sightlines['DEC'], unit='deg')
idx, d2d, d3d = match_coordinates_sky(s_coord, ml_coord, nthneighbor=1)
used = d2d < 1.*u.arcsec
for iidx in np.where(~used)[0]:
print("Sightline RA={:g}, DEC={:g} was not used".format(sdss_survey.sightlines['RA'][iidx],
sdss_survey.sightlines['DEC'][iidx]))
# Add plate/fiber to statistical DLAs
if add_pf:
dla_coord = sdss_survey.coord
idx2, d2d, d3d = match_coordinates_sky(dla_coord, s_coord, nthneighbor=1)
if np.min(d2d.to('arcsec').value) > 1.:
raise ValueError("Bad match to sightlines")
for jj,igd in enumerate(np.where(sdss_survey.mask)[0]):
dla = sdss_survey._abs_sys[igd]
try:
dla.plate = sdss_survey.sightlines['PLATE'][idx2[jj]]
except IndexError:
pdb.set_trace()
dla.fiber = sdss_survey.sightlines[fkey][idx2[jj]]
# Finish
ml_survey._abs_sys = systems
if debug:
ml2_coord = ml_survey.coord
minsep = np.min(sv_coord.separation(ml2_coord))
minsep2 = np.min(sv_coord.separation(s_coord))
tmp = sdss_survey.sightlines[used]
t_coord = SkyCoord(ra=tmp['RA'], dec=tmp['DEC'], unit='deg')
minsep3 = np.min(sv_coord.separation(t_coord))
pdb.set_trace()
ml_survey.sightlines = sdss_survey.sightlines[used]
for key in idict.keys():
ml_tbl[key] = idict[key]
ml_survey.ml_tbl = ml_tbl
# Return
return ml_survey
