def main(pargs):
""" Run
"""
import warnings
from IPython import embed
from linetools import utils as ltu
from tennis import tourney
from tennis import io as tennis_io
# Load seeds
seeds = tennis_io.load_seeding(pargs.names)
# Generate
matches = tourney.generate_match_ups(seeds)
# Write to JSON
if pargs.json_file is not None:
ltu.savejson(pargs.json_file,
matches,
overwrite=True,
easy_to_read=True)
print("Wrote: {:s}".format(pargs.json_file))
# Table
if pargs.table_file is not None:
mtch_tbl = tourney.table_from_matches(matches)
mtch_tbl.write(pargs.table_file, format='csv')
def write_to_json(self, outfile):
# Generate the dict
igms_dict = self.to_dict()
# Jsonify
clean_dict = ltu.jsonify(igms_dict)
# Write
ltu.savejson(outfile, clean_dict, overwrite=True)
def save(self, idx):
# Update dict
self.update_dict(idx)
# Save
cjson = ltu.jsonify(self.zdict)
ltu.savejson(self.outfile, cjson, overwrite=True, easy_to_read=True)
print("Wrote: {:s}".format(self.outfile))
def test_write_sngl():
import io, json
from linetools import utils as ltu
# Class
cos_halos = COSHalos(fits_path=data_path(''),
cdir=data_path(''),
load=False)
# Load
cos_halos.load_single_fits(('J0950+4831', '177_27'))
# Write to JSON
cdict = cos_halos.cgm_abs[0].to_dict()
ltu.savejson('tmp.json', cdict, overwrite=True)
def test_dicts():
# Init HI Lya
abslin = AbsLine(1215.6700*u.AA)
adict = abslin.to_dict()
assert isinstance(adict, dict)
# Write
#pdb.set_trace()
ltu.savejson('tmp.json', adict, overwrite=True)
# Read
newdict = ltu.loadjson('tmp.json')
newlin = SpectralLine.from_dict(newdict)
assert newlin.name == 'HI 1215'
def test_dicts():
# Init Halpha
emisslin = EmLine(6564.613*u.AA)
emisslin.analy['spec'] = 'tmp.fits'
edict = emisslin.to_dict()
assert isinstance(edict, dict)
# Write
ltu.savejson('tmp.json', edict, overwrite=True)
# Read
newdict = ltu.loadjson('tmp.json')
newlin = SpectralLine.from_dict(newdict)
assert newlin.name == 'Halpha'
assert newlin.ltype == 'Em'
def test_dicts():
# Init Halpha
emisslin = EmLine(6564.613 * u.AA)
emisslin.analy['spec'] = 'tmp.fits'
edict = emisslin.to_dict()
assert isinstance(edict, dict)
# Write
ltu.savejson('tmp.json', edict, overwrite=True)
# Read
newdict = ltu.loadjson('tmp.json')
newlin = SpectralLine.from_dict(newdict)
assert newlin.name == 'Halpha'
assert newlin.ltype == 'Em'
def test_to_dict():
from linetools import utils as ltu
radec = (125 * u.deg, 45.2 * u.deg)
gal = Galaxy(radec, z=0.3)
radec_qso = (125 * u.deg, 45.203 * u.deg)
igmsys = IGMSystem(radec_qso,
gal.z, [-500, 500] * u.km / u.s,
abs_type='CGM')
# Instantiate
cgmabs = CGMAbsSys(gal, igmsys)
# Test
cdict = cgmabs.to_dict()
ltu.savejson('tmp.json', cdict, overwrite=True)
def get_tslits_nires(flat_files,
user_settings=par,
gingashow=True,
tilt_root='tilt_nires'):
"""Precess flat files and get titlts for NIRES
"""
# Process flat images
tImage = traceimage.TraceImage(spectrograph,
file_list=flat_files,
par=par['calibrations']['traceframe'])
tflat = tImage.process(bias_subtract='overscan',
trim=False)
mstrace = tflat.copy()
# Define pixlocn and bpm
pixlocn = pixels.gen_pixloc(tImage.stack.shape)
bpm = spectrograph.bpm(shape=tflat.shape, det=1)
# Instantiate Trace
tSlits = traceslits.TraceSlits(mstrace,
pixlocn,
par=par['calibrations']['slits'],
binbpx=bpm)
tslits_dict = tSlits.run(plate_scale = 0.123)
if gingashow:
# Look at what TraceSlits was actually trying to trace
viewer, ch = ginga.show_image(tSlits.edgearr)
# Look at the sawtooth convolved image
viewer, ch = ginga.show_image(tSlits.siglev)
tmp = tSlits.edgearr * 100.
tmp[np.where(tmp == 0.)] = 1.
ginga.show_image(tSlits.mstrace * tmp)
ginga.show_slits(viewer,
ch,
tSlits.lcen,
tSlits.rcen,
slit_ids=np.arange(tSlits.lcen.shape[1]) + 1,
pstep=50)
if tilt_root is not None:
# Write dict on a json file
jdict = ltu.jsonify(tslits_dict.copy())
ltu.savejson(tilt_root + '.json', jdict, overwrite=True, indent=None, easy_to_read=True)
print("Wrote: {:s}".format(tilt_root + '.json'))
return tslits_dict
def write_json(self, outfil=None):
""" Generate a JSON file from the system
Returns
-------
"""
# Generate the dict
odict = self.to_dict()
# Write
if outfil is None:
outfil = self.name+'.json'
ltu.savejson(outfil, odict, overwrite=True, easy_to_read=True)
# Finish
print("Wrote {:s} system to {:s} file".format(self.name, outfil))
def write_json(self, outfil=None):
""" Generate a JSON file from the system
Returns
-------
"""
# Generate the dict
odict = self.to_dict()
# Write
if outfil is None:
outfil = self.name+'.json'
ltu.savejson(outfil, odict, overwrite=True, easy_to_read=True)
# Finish
print("Wrote {:s} system to {:s} file".format(self.name, outfil))
def write_bg_regions(bg_region, outfile):
""" Write background regions to a simple JSON file
Parameters
----------
bg_region : dict
outfile : str
Returns
-------
"""
jdict = ltu.jsonify(bg_region)
# Write
ltu.savejson(outfile, jdict, easy_to_read=True, overwrite=True)
print("Wrote Background Regions to {:s}", outfile)
def write(self, outfile, overwrite=True):
""" Write to a JSON file
Parameters
----------
outfile : str
overwrite : bool, optional
Returns
-------
"""
# Generate the dict
cdict = self.to_dict()
# Write
ltu.savejson(outfile, cdict, overwrite=overwrite, easy_to_read=True)
print("Wrote AbsComponent to {:s}".format(outfile))
def write(self, outfile, overwrite=True):
""" Write to a JSON file
Parameters
----------
outfile : str
overwrite : bool, optional
Returns
-------
"""
# Generate the dict
cdict = self.to_dict()
# Write
ltu.savejson(outfile, cdict, overwrite=overwrite, easy_to_read=True)
print("Wrote AbsComponent to {:s}".format(outfile))
def write_traces(obj, arc, outfile):
""" Write a simple JSON file
Parameters
----------
obj : float
arc : float
outfile : str
Returns
-------
"""
tdict = dict(obj=obj, arc=arc)
jdict = ltu.jsonify(tdict)
# Write
ltu.savejson(outfile, jdict, easy_to_read=True, overwrite=True)
print("Wrote Traces to {:s}", outfile)
def write_json(self, outfil=None, overwrite=True):
""" Generate a JSON file from the system
Parameters
----------
outfil : str, optional
Output filename; generated from system name if not provided
overwrite : bool, optional
Overwrite?
"""
# Generate the dict
odict = self.to_dict()
# Write
if outfil is None:
outfil = self.name+'.json'
ltu.savejson(outfil, odict, overwrite=overwrite, easy_to_read=True)
# Finish
print("Wrote {:s} system to {:s} file".format(self.name, outfil))
def test_dicts():
# Init HI Lya
abslin = AbsLine(1215.6700*u.AA)
abslin.analy['spec'] = 'tmp.fits'
adict = abslin.to_dict()
assert isinstance(adict, dict)
# Write
#pdb.set_trace()
ltu.savejson('tmp.json', adict, overwrite=True)
# Read
newdict = ltu.loadjson('tmp.json')
newlin = SpectralLine.from_dict(newdict)
assert newlin.name == 'HI 1215'
# Old dict for compatability
newdict.pop('limits')
newdict['analy']['vlim'] = [-150,150]*u.km/u.s
newdict['attrib']['z'] = 0.5
tmp3 = SpectralLine.from_dict(newdict)
assert newlin.name == 'HI 1215'
def test_dicts():
# Init HI Lya
abslin = AbsLine(1215.6700 * u.AA)
abslin.analy['spec'] = 'tmp.fits'
adict = abslin.to_dict()
assert isinstance(adict, dict)
# Write
#pdb.set_trace()
ltu.savejson('tmp.json', adict, overwrite=True)
# Read
newdict = ltu.loadjson('tmp.json')
newlin = SpectralLine.from_dict(newdict)
assert newlin.name == 'HI 1215'
# Old dict for compatability
newdict.pop('limits')
newdict['analy']['vlim'] = [-150, 150] * u.km / u.s
newdict['attrib']['z'] = 0.5
tmp3 = SpectralLine.from_dict(newdict)
assert newlin.name == 'HI 1215'
def to_json(self, outfile, overwrite=True):
""" Generates a JSON file of the survey
Parameters
----------
outfil : str
"""
survey_dict = OrderedDict()
# Loop on systems
for cgm_abs in self.cgm_abs:
# Dict from copy
cdict = cgm_abs.to_dict()
# Use galaxy name for key; Should be unique
survey_dict[cgm_abs.galaxy.name + '_' +
cgm_abs.igm_sys.name] = cdict.copy()
# JSON
clean_dict = ltu.jsonify(survey_dict)
ltu.savejson(outfile, clean_dict, overwrite=overwrite)
print("Wrote: {:s}".format(outfile))
print("You may now wish to compress it..")
def update_dla_fits(new_fits):
import datetime
import getpass
# Load existing
dla_fits, fit_file = load_dla_fits()
# Write fit
date = str(datetime.date.today().strftime('%Y-%b-%d'))
user = getpass.getuser()
#
for key in new_fits:
# Add
if key not in dla_fits.keys():
dla_fits[key] = {}
for subkey in new_fits[key]:
dla_fits[key][subkey] = new_fits[key][subkey]
dla_fits[key][subkey]['CreationDate'] = date
dla_fits[key][subkey]['User'] = user
# Write
pdb.set_trace()
jdfits = ltu.jsonify(dla_fits)
ltu.savejson(fit_file, jdfits, easy_to_read=True, overwrite=True)
print("Wrote: {:s}".format(fit_file))
def test_save_load_json():
tmp_dict = dict(a=1, b=2, c='adsf')
# Write
ltu.savejson('tmp.json', tmp_dict, overwrite=True)
# Load
new_dict = ltu.loadjson('tmp.json')
assert new_dict['a'] == 1
# Write with gzip
ltu.savejson('tmp.json.gz', tmp_dict, overwrite=True)
# Load
new_dict = ltu.loadjson('tmp.json.gz')
assert new_dict['a'] == 1
# Write with easy to read
ltu.savejson('tmp2.json', tmp_dict, overwrite=True, easy_to_read=True)
new_dict2 = ltu.loadjson('tmp2.json')
assert new_dict2['a'] == 1
def test_save_load_json():
tmp_dict = dict(a=1, b=2, c='adsf')
# Write
ltu.savejson('tmp.json', tmp_dict, overwrite=True)
# Load
new_dict = ltu.loadjson('tmp.json')
assert new_dict['a'] == 1
# Write with gzip
ltu.savejson('tmp.json.gz', tmp_dict, overwrite=True)
# Load
new_dict = ltu.loadjson('tmp.json.gz')
assert new_dict['a'] == 1
# Write with easy to read
ltu.savejson('tmp2.json', tmp_dict, overwrite=True, easy_to_read=True)
new_dict2 = ltu.loadjson('tmp2.json')
assert new_dict2['a'] == 1
def write_to_igmguesses(self,
outfile,
fwhm=3.,
specfilename=None,
creator=None,
instrument='unknown',
altname='unknown',
date=None,
overwrite=False):
import json
"""
Writes an IGMGuesses formatted JSON file
Parameters
----------
outfile : str
Name of the IGMGuesses JSON file to write to.
fwhm : int
FWHM for IGMguesses
specfilename : str
Name of the spectrum file these guesses are associated to.
altname : str
Alternative name for the sightline. e.g. 3C273
creator : str
Name of the person who is creating the file. Important for tracking.
instrument : str
String indicating the instrument and its configuration associated to the specfilename.
e.g. HST/COS/G130M+G160M/LP2
overwrite : bool
Wthether to overwrite output
Returns
-------
"""
import datetime
# Slurp IGMGuesses component attributes
from pyigm.guis.igmguesses import comp_init_attrib
from linetools.isgm.abscomponent import AbsComponent
tmp = AbsComponent((10.0 * u.deg, 45 * u.deg), (14, 2), 1.0,
[-300, 300] * u.km / u.s)
comp_init_attrib(tmp)
igmg_akeys = list(tmp.attrib.keys())
# components
comp_list = self._components
coord_ref = comp_list[0].coord
if date is None:
date = str(datetime.date.today().strftime('%Y-%b-%d'))
# spec_file, meta
if hasattr(self, 'igmg_dict'):
spec_file = self.igmg_dict['spec_file']
meta = self.igmg_dict['meta']
# Updates
meta['Date'] = date
if creator is not None:
meta['Creator'] = creator
#
fwhm = self.igmg_dict['fwhm']
else:
spec_file = specfilename
# coordinates and meta
RA = coord_ref.ra.to('deg').value
DEC = coord_ref.dec.to('deg').value
jname = ltu.name_from_coord(coord_ref, precision=(2, 1))
if self.zem is None:
zem = 0. # IGMGuesses rules
else:
zem = self.zem
if creator is None:
creator = 'unknown'
meta = {
'RA': RA,
'DEC': DEC,
'ALTNAME': altname,
'zem': zem,
'Creator': creator,
'Instrument': instrument,
'Date': date,
'JNAME': jname
}
# Create dict of the components
out_dict = dict(cmps={},
spec_file=spec_file,
fwhm=fwhm,
bad_pixels=[],
meta=meta)
for comp in comp_list:
key = comp.name
out_dict['cmps'][key] = comp.to_dict()
# import pdb; pdb.set_trace()
# check coordinate
if comp.coord != coord_ref:
raise ValueError(
"All AbsComponent objects must have the same coordinates!")
out_dict['cmps'][key]['zcomp'] = comp.zcomp
# IGMGuesses specific component attr
for igm_key in [
'zfit', 'Nfit', 'bfit', 'wrest', 'mask_abslines', 'vlim'
]:
out_dict['cmps'][key][igm_key] = comp.igmg_attrib['top_level'][
igm_key]
# IGMGuesses attribute dict
out_dict['cmps'][key]['attrib'] = {}
for igm_key in igmg_akeys:
try:
out_dict['cmps'][key]['attrib'][
igm_key] = comp.igmg_attrib[igm_key]
except KeyError:
out_dict['cmps'][key]['attrib'][igm_key] = comp.attrib[
igm_key]
#out_dict['cmps'][key]['vlim'] = list(comp.vlim.value)
out_dict['cmps'][key]['reliability'] = str(comp.reliability)
out_dict['cmps'][key]['comment'] = str(comp.comment)
# Compatability on sig_logN
out_dict['cmps'][key]['attrib']['sig_logN'] = comp.attrib[
'sig_logN'][0]
# JSONify
gd_dict = ltu.jsonify(out_dict)
# Write file
ltu.savejson(outfile,
gd_dict,
overwrite=overwrite,
sort_keys=True,
indent=4,
separators=(',', ': '))
print('Wrote: {:s}'.format(outfile))
def chk_pn_dla_to_ml(ml_dlasurvey=None, ml_llssurvey=None, dz_toler=0.015, outfile='vette_dr7_pn.json'):
""" Compare results of Noterdaeme to ML
Save to JSON file
"""
# Load ML
if (ml_dlasurvey is None) or (ml_llssurvey is None):
ml_llssurvey, ml_dlasurvey = load_ml_dr7()
# Load PN
pn_dr7_file = '../Analysis/noterdaeme_dr7.fits'
pn_dr7 = Table.read(pn_dr7_file)
# Use coord to efficiently deal with sightlines
ml_coord = SkyCoord(ra=ml_dlasurvey.sightlines['RA'], dec=ml_dlasurvey.sightlines['DEC'], unit='deg')
pn_coord = SkyCoord(ra=pn_dr7['_RA'], dec=pn_dr7['_DE'], unit='deg')
idx, d2d, d3d = match_coordinates_sky(pn_coord, ml_coord, nthneighbor=1)
in_ml = d2d < 2*u.arcsec
print("{:d} of the PN sightlines were covered by ML out of {:d}".format(np.sum(in_ml), len(pn_dr7)))
# Cut
cut_pn = pn_dr7[in_ml]
# Loop on PN DLAs and save indices of the matches
pn_ml_idx = np.zeros(len(cut_pn)).astype(int) - 1
for ii,pnrow in enumerate(cut_pn):
if pnrow['logN_HI_'] >= 20.3:
dla_mts = np.where((ml_dlasurvey.plate == pnrow['Plate']) & (ml_dlasurvey.fiber == pnrow['Fiber']))[0]
nmt = len(dla_mts)
if nmt == 0: # No match
# Check for LLS
lls_mts = np.where((ml_llssurvey.plate == pnrow['Plate']) & (ml_llssurvey.fiber == pnrow['Fiber']))[0]
nmt2 = len(lls_mts)
if nmt2 == 0: # No match
pass
else:
zML = ml_llssurvey.zabs[lls_mts] # Redshifts of all DLAs on the sightline in ML
zdiff = np.abs(pnrow['zabs']-zML)
if np.min(zdiff) < dz_toler:
pn_ml_idx[ii] = -9 # SLLS match
else:
zML = ml_dlasurvey.zabs[dla_mts] # Redshifts of all DLAs on the sightline in ML
zdiff = np.abs(pnrow['zabs']-zML)
if np.min(zdiff) < dz_toler:
#print("Match on {:d}!".format(ii))
# Match
imin = np.argmin(zdiff)
pn_ml_idx[ii] = dla_mts[imin]
else:
pn_ml_idx[ii] = -99 # Not a PN DLA
# Stats on matches
'''
gdm = pn_ml_idx >= 0
pdb.set_trace()
dz = cut_pn['zabs'][gdm]-ml_dlasurvey.zabs[pn_ml_idx[gdm]]
dNHI = cut_pn['logN_HI_'][gdm]-ml_dlasurvey.NHI[pn_ml_idx[gdm]]
plt.clf()
#plt.hist(dz)
plt.hist(dNHI)
plt.show()
'''
# PN not matched by ML?
misses = (pn_ml_idx == -1)
pn_missed = cut_pn[misses]
# Write high NHI systems to disk
high_NHI = pn_missed['logN_HI_'] > 20.8
pn_missed[['QSO','Plate','Fiber', 'zem', 'zabs', 'Flag', 'logN_HI_']][high_NHI].write("N09_missed_highNHI.ascii", format='ascii.fixed_width', overwrite=True)
# ML not matched by PN?
ml_dla_coords = ml_dlasurvey.coords
idx2, d2d2, d3d = match_coordinates_sky(ml_dla_coords, pn_coord, nthneighbor=1)
not_in_pn = d2d2 > 2*u.arcsec # This doesn't check zabs!!
tmp_tbl = Table()
for key in ['plate', 'fiber', 'zabs', 'NHI', 'confidence']:
tmp_tbl[key] = getattr(ml_dlasurvey, key)
# Save
out_dict = {}
out_dict['in_ml'] = in_ml
out_dict['pn_idx'] = pn_ml_idx # -1 are misses, -99 are not DLAs in PN
out_dict['not_in_pn'] = np.where(not_in_pn)[0]
ltu.savejson(outfile, ltu.jsonify(out_dict), overwrite=True)
print("Wrote: {:s}".format(outfile))
# Line list
CuI = waveio.load_line_list('CuI', use_ion=True, NIST=True)
ArI = waveio.load_line_list('ArI', use_ion=True, NIST=True)
ArII = waveio.load_line_list('ArII', use_ion=True, NIST=True)
llist = vstack([CuI, ArI, ArII])
arcparam['llist'] = llist
# Simple calibration
final_fit = arc.simple_calib(dummy,
arcparam,
spec,
IDpixels=IDpixels,
IDwaves=IDwaves,
nfitpix=9) #, sigdetect=5.) #sigdetect=7.)
arc.arc_fit_qa(None, final_fit, None, outfile='GMOS_R400_wave.png')
jdict = ltu.jsonify(final_fit)
ltu.savejson(outfile, jdict, overwrite=True)
pdb.set_trace()
# Arc fitter
lines = ['CuI', 'ArI', 'ArII']
min_ampl = 1000.
arcfitter = autoid.General(spec.reshape((spec.size, 1)),
lines,
min_ampl=min_ampl,
rms_threshold=0.2,
nonlinear_counts=80000.)
pdb.set_trace()
def score_ml_test(dz_toler=0.015,
outfile='vette_10k.json',
test_file='data/test_dlas_96629_10000.json.gz',
pred_file='data/test_dlas_96629_predictions.json.gz'):
# Load Test
test_dlas = test_to_tbl(test_file)
ntest = len(test_dlas)
# Load ML
ml_abs = pred_to_tbl(pred_file)
# Loop on test DLAs and save indices of the matches
test_ml_idx = np.zeros(ntest).astype(int) - 99999
for ii in range(ntest):
# Match to ML sl
in_sl = np.where(ml_abs['ids'] == test_dlas['ids'][ii])[0]
dla_mts = np.where(
np.abs(ml_abs['zabs'][in_sl] -
test_dlas['zabs'][ii]) < dz_toler)[0]
nmt = len(dla_mts)
if nmt == 0: # No match within dz
pass
elif nmt == 1: # No match
if ml_abs['NHI'][in_sl][dla_mts[0]] > 20.2999:
test_ml_idx[ii] = in_sl[dla_mts[0]]
else:
test_ml_idx[ii] = -1 * in_sl[dla_mts[0]]
else: # Very rarely the ML identifies two DLAs in the window
print("Double hit in test DLA {:d}".format(ii))
imin = np.argmin(
np.abs(ml_abs['zabs'][in_sl] - test_dlas['zabs'][ii]))
test_ml_idx[ii] = in_sl[imin]
match = test_ml_idx >= 0
print("There were {:d} DLAs recovered out of {:d}".format(
np.sum(match), ntest))
# Write out misses
misses = np.where(test_ml_idx == -99999)[0]
print("There were {:d} DLAs missed altogether".format(len(misses)))
mtbl = Table()
for key in ['sl', 'NHI', 'zabs']:
mtbl[key] = test_dlas[key][misses]
mtbl.write('test_misses.ascii', format='ascii.fixed_width', overwrite=True)
# Write out SLLS
sllss = np.where((test_ml_idx < 0) & (test_ml_idx != -99999))[0]
print("There were {:d} DLAs recovered as SLLS".format(len(sllss)))
stbl = Table()
for key in ['sl', 'NHI', 'zabs']:
stbl[key] = test_dlas[key][sllss]
mtbl.write('test_slls.ascii', format='ascii.fixed_width', overwrite=True)
# Save
out_dict = {}
out_dict[
'test_idx'] = test_ml_idx # -1 are misses, -99 are not DLAs in PN, -9 are SLLS
ltu.savejson(outfile, ltu.jsonify(out_dict), overwrite=True)
# Stats on dz
dz = ml_abs['zabs'][test_ml_idx[match]] - test_dlas['zabs'][match]
print("Median dz = {} and sigma(dz)= {}".format(np.median(dz), np.std(dz)))
def make_set(ntrain,
slines,
outroot=None,
tol=1 * u.arcsec,
igmsp_survey='SDSS_DR7',
frac_without=0.,
seed=1234,
zmin=None,
zmax=4.5,
high=False,
slls=False,
mix=False,
low_s2n=False):
""" Generate a training set
Parameters
----------
ntrain : int
Number of training sightlines to generate
slines : Table
Table of sightlines without DLAs (usually from SDSS or BOSS)
igmsp_survey : str, optional
Dataset name for spectra
frac_without : float, optional
Fraction of sightlines (on average) without a DLA
seed : int, optional
outroot : str, optional
Root for output filenames
root+'.fits' for spectra
root+'.json' for DLA info
zmin : float, optional
Minimum redshift for training; defaults to min(slines['ZEM'])
zmax : float, optional
Maximum redshift to train on
mix : bool, optional
Mix of SLLS and DLAs
low_s2n : bool, optional
Reduce the S/N artificially, i.e. add noise
Returns
-------
"""
from linetools.spectra.utils import collate
# Init and checks
igmsp = IgmSpec()
assert igmsp_survey in igmsp.groups
rstate = np.random.RandomState(seed)
rfrac = rstate.random_sample(ntrain)
if zmin is None:
zmin = np.min(slines['ZEM'])
rzem = zmin + rstate.random_sample(ntrain) * (zmax - zmin)
fNHI = init_fNHI(slls=slls, mix=mix, high=high)
all_spec = []
full_dict = {}
# Begin looping
for qq in range(ntrain):
print("qq = {:d}".format(qq))
full_dict[qq] = {}
# Grab sightline
isl = np.argmin(np.abs(slines['ZEM'] - rzem[qq]))
full_dict[qq]['sl'] = isl # sightline
specl, meta = igmsp.spectra_from_coord(
(slines['RA'][isl], slines['DEC'][isl]),
groups=['SDSS_DR7'],
tol=tol,
verbose=False)
assert len(meta) == 1
spec = specl
# Meta data for header
mdict = {}
for key in meta.keys():
mdict[key] = meta[key][0]
mhead = Header(mdict)
# Clear?
if rfrac[qq] < frac_without:
spec.meta['headers'][0] = mdict.copy() #mhead
all_spec.append(spec)
full_dict[qq]['nDLA'] = 0
continue
# Insert at least one DLA
spec, dlas = insert_dlas(spec,
mhead['zem_GROUP'],
rstate=rstate,
fNHI=fNHI,
slls=slls,
mix=mix,
high=high,
low_s2n=low_s2n)
spec.meta['headers'][0] = mdict.copy() #mhead
all_spec.append(spec)
full_dict[qq]['nDLA'] = len(dlas)
for kk, dla in enumerate(dlas):
full_dict[qq][kk] = {}
full_dict[qq][kk]['NHI'] = dla.NHI
full_dict[qq][kk]['zabs'] = dla.zabs
# Generate one object
final_spec = collate(all_spec)
# Write?
if outroot is not None:
# Spectra
final_spec.write_to_hdf5(outroot + '.hdf5')
# Dict -> JSON
gdict = ltu.jsonify(full_dict)
ltu.savejson(outroot + '.json', gdict,
overwrite=True) #, easy_to_read=True)
# Return
return final_spec, full_dict
def score_ml_test(
dz_toler=0.015,
outfile='vette_mix.json',
test_file='data/mix_test_23559_10000.json',
pred_file='../Analysis/visuals_mix/mixed_predictions.json.gz'):
"""
Parameters
----------
dz_toler
outfile
test_file
pred_file
Returns
-------
writes
"""
# Load Test
mix_abs = test_to_tbl(test_file)
mix_dlas = mix_abs[mix_abs['NHI'] >= 20.3]
ntest = len(mix_dlas)
mix_slls = mix_abs[mix_abs['NHI'] < 20.3]
n_slls = len(mix_slls)
# Load ML
ml_abs = pred_to_tbl(pred_file)
ml_dlas = ml_abs['NHI'] >= 20.3
ml_slls = ml_abs['NHI'] < 20.3
# Loop on test DLAs and save indices of the matches
test_ml_idx = np.zeros(ntest).astype(int) - 99999
for ii in range(ntest):
# Match to ML sl
in_sl = np.where(ml_abs['ids'] == mix_dlas['ids'][ii])[0]
dla_mts = np.where(
np.abs(ml_abs['zabs'][in_sl] - mix_dlas['zabs'][ii]) < dz_toler)[0]
nmt = len(dla_mts)
if nmt == 0: # No match within dz
pass
elif nmt == 1: # One match
if ml_abs['NHI'][in_sl][dla_mts[0]] > 20.2999:
test_ml_idx[ii] = in_sl[dla_mts[0]]
else:
test_ml_idx[ii] = -1 * in_sl[dla_mts[0]]
else: # Very rarely the ML identifies two DLAs in the window
print("Double hit in mix DLA {:d}".format(ii))
imin = np.argmin(
np.abs(ml_abs['zabs'][in_sl] - mix_dlas['zabs'][ii]))
test_ml_idx[ii] = in_sl[imin]
match = test_ml_idx >= 0
print("There were {:d} DLAs discovered by the CNN".format(np.sum(ml_dlas)))
print("There were {:d} DLAs recovered out of {:d}".format(
np.sum(match), ntest))
print("There were {:d} false positive DLAs".format(
np.sum(ml_dlas) - np.sum(match)))
# Write out misses
misses = np.where(test_ml_idx == -99999)[0]
print("There were {:d} DLAs missed altogether (false negatives)".format(
len(misses)))
mtbl = Table()
for key in ['sl', 'NHI', 'zabs']:
mtbl[key] = mix_dlas[key][misses]
mtbl.write('mix_misses.ascii', format='ascii.fixed_width', overwrite=True)
# Write out SLLS
sllss = np.where((test_ml_idx < 0) & (test_ml_idx != -99999))[0]
print("There were {:d} DLAs recovered as SLLS".format(len(sllss)))
stbl = Table()
for key in ['sl', 'NHI', 'zabs']:
stbl[key] = mix_dlas[key][sllss]
mtbl.write('mix_slls.ascii', format='ascii.fixed_width', overwrite=True)
# Loop on test SLLS and save indices of the matches
test_ml_slls_idx = np.zeros(n_slls).astype(int) - 99999
for ii in range(n_slls):
# Match to ML sl
in_sl = np.where(ml_abs['ids'] == mix_slls['ids'][ii])[0]
slls_mts = np.where(
np.abs(ml_abs['zabs'][in_sl] - mix_slls['zabs'][ii]) < dz_toler)[0]
nmt = len(slls_mts)
if nmt == 0: # No match within dz
pass
elif nmt == 1: # One match
if ml_abs['NHI'][in_sl][slls_mts[0]] > 20.2999:
test_ml_slls_idx[ii] = -1 * in_sl[slls_mts[0]]
else:
test_ml_slls_idx[ii] = in_sl[slls_mts[0]]
else: # Very rarely the ML identifies two DLAs in the windo
print("Double hit in mix DLA {:d}".format(ii))
imin = np.argmin(
np.abs(ml_abs['zabs'][in_sl] - mix_slls['zabs'][ii]))
test_ml_slls_idx[ii] = in_sl[imin]
slls_match = test_ml_slls_idx >= 0
print("There were {:d} SLLS discovered by the CNN".format(np.sum(ml_slls)))
print("There were {:d} SLLS recovered out of {:d}".format(
np.sum(slls_match), n_slls))
print("There were {:d} false positive SLLS".format(
np.sum(ml_slls) - np.sum(slls_match)))
# Save
out_dict = {}
out_dict[
'test_idx'] = test_ml_idx # -1 are misses, -99 are not DLAs in PN, -9 are SLLS
ltu.savejson(outfile, ltu.jsonify(out_dict), overwrite=True)
# Stats on dz
dz = ml_abs['zabs'][test_ml_idx[match]] - mix_dlas['zabs'][match]
print("Median dz = {} and sigma(dz)= {}".format(np.median(dz), np.std(dz)))
def chk_dr5_dla_to_ml(ml_dlasurvey=None, ml_llssurvey=None, dz_toler=0.015,
outfile='vette_dr5.json', write_again=True):
# Load ML
if (ml_dlasurvey is None) or (ml_llssurvey is None):
ml_llssurvey, ml_dlasurvey = load_ml_dr7()
# Load DR5
dr5 = DLASurvey.load_SDSS_DR5() # This is the statistical sample
# Use coord to efficiently deal with sightlines
ml_coord = SkyCoord(ra=ml_dlasurvey.sightlines['RA'], dec=ml_dlasurvey.sightlines['DEC'], unit='deg')
dr5_coord = SkyCoord(ra=dr5.sightlines['RA'], dec=dr5.sightlines['DEC'], unit='deg')
idx, d2d, d3d = match_coordinates_sky(dr5_coord, ml_coord, nthneighbor=1)
in_ml = d2d < 2*u.arcsec
print("{:d} of the DR5 sightlines were covered by ML out of {:d}".format(np.sum(in_ml), len(dr5.sightlines)))
# 7477 sightlines out of 7482
# Cut down
dr5.sightlines = dr5.sightlines[in_ml]
new_mask = dla_stat(dr5, dr5.sightlines) # 737 good DLAs
dr5.mask = new_mask
dr5_dla_coord = dr5.coord
dr5_dla_zabs = dr5.zabs
ndr5 = len(dr5_dla_coord)
ml_dla_coord = ml_dlasurvey.coords
ml_lls_coord = ml_llssurvey.coords
# Loop on DR5 DLAs and save indices of the matches
dr5_ml_idx = np.zeros(ndr5).astype(int) - 1
for ii in range(ndr5):
# Match to ML
dla_mts = np.where(dr5_dla_coord[ii].separation(ml_dla_coord) < 2*u.arcsec)[0]
nmt = len(dla_mts)
if nmt == 0: # No match
# Check for LLS
lls_mts = np.where(dr5_dla_coord[ii].separation(ml_lls_coord) < 2*u.arcsec)[0]
nmt2 = len(lls_mts)
if nmt2 == 0: # No match
pass
else:
zML = ml_llssurvey.zabs[lls_mts] # Redshifts of all DLAs on the sightline in ML
zdiff = np.abs(dr5_dla_zabs[ii]-zML)
if np.min(zdiff) < dz_toler:
dr5_ml_idx[ii] = -9 # SLLS match
else:
zML = ml_dlasurvey.zabs[dla_mts] # Redshifts of all DLAs on the sightline in ML
zdiff = np.abs(dr5_dla_zabs[ii]-zML)
if np.min(zdiff) < dz_toler:
#print("Match on {:d}!".format(ii))
# Match
imin = np.argmin(zdiff)
dr5_ml_idx[ii] = dla_mts[imin]
else: # Check for LLS
lls_mts = np.where(dr5_dla_coord[ii].separation(ml_lls_coord) < 2*u.arcsec)[0]
nmt2 = len(lls_mts)
if nmt2 == 0: # No match
pass
else:
zML = ml_llssurvey.zabs[lls_mts] # Redshifts of all DLAs on the sightline in ML
zdiff = np.abs(dr5_dla_zabs[ii]-zML)
if np.min(zdiff) < dz_toler:
dr5_ml_idx[ii] = -9 # SLLS match
dr5_coord = SkyCoord(ra=dr5.sightlines['RA'], dec=dr5.sightlines['DEC'], unit='deg')
# Write out misses
misses = np.where(dr5_ml_idx == -1)[0]
plates, fibers = [], []
for miss in misses:
imin = np.argmin(dr5_dla_coord[miss].separation(dr5_coord))
plates.append(dr5.sightlines['PLATE'][imin])
fibers.append(dr5.sightlines['FIB'][imin])
mtbl = Table()
mtbl['PLATE'] = plates
mtbl['FIBER'] = fibers
mtbl['NHI'] = dr5.NHI[misses]
mtbl['zabs'] = dr5.zabs[misses]
if write_again:
mtbl.write('DR5_misses.ascii', format='ascii.fixed_width', overwrite=True)
# Write out SLLS
sllss = np.where(dr5_ml_idx == -9)[0]
plates, fibers = [], []
for slls in sllss:
imin = np.argmin(dr5_dla_coord[slls].separation(dr5_coord))
plates.append(dr5.sightlines['PLATE'][imin])
fibers.append(dr5.sightlines['FIB'][imin])
mtbl = Table()
mtbl['PLATE'] = plates
mtbl['FIBER'] = fibers
mtbl['NHI'] = dr5.NHI[sllss]
mtbl['zabs'] = dr5.zabs[sllss]
if write_again:
mtbl.write('DR5_SLLS.ascii', format='ascii.fixed_width', overwrite=True)
pdb.set_trace()
# ML not matched by PW09?
ml_dla_coords = ml_dlasurvey.coords
idx2, d2d2, d3d = match_coordinates_sky(ml_dla_coords, dr5_dla_coord, nthneighbor=1)
not_in_dr5 = d2d2 > 2*u.arcsec # This doesn't match redshifts!
might_be_in_dr5 = np.where(~not_in_dr5)[0]
others_not_in = [] # this is some painful book-keeping
for idx in might_be_in_dr5: # Matching redshifts..
imt = ml_dla_coord[idx].separation(dr5_dla_coord) < 2*u.arcsec
# Match on dztoler
if np.min(np.abs(ml_dlasurvey.zabs[idx]-dr5.zabs[imt])) > dz_toler:
others_not_in.append(idx)
# Save
out_dict = {}
out_dict['in_ml'] = in_ml
out_dict['dr5_idx'] = dr5_ml_idx # -1 are misses, -9 are SLLS
out_dict['not_in_dr5'] = np.concatenate([np.where(not_in_dr5)[0], np.array(others_not_in)])
ltu.savejson(outfile, ltu.jsonify(out_dict), overwrite=True)
def calib_all(self, run=True):
"""
Create calibrations for all setups
This will not crash if not all of the standard set of files are not provided
Args:
run (bool, optional): If False, only print the calib names and do
not actually run. Only used with the pypeit_parse_calib_id script
Returns:
dict: A simple dict summarizing the calibration names
"""
calib_dict = {}
self.tstart = time.time()
# Frame indices
frame_indx = np.arange(len(self.fitstbl))
for i in range(self.fitstbl.n_calib_groups):
# 1-indexed calib number
calib_grp = str(i + 1)
# Find all the frames in this calibration group
in_grp = self.fitstbl.find_calib_group(i)
grp_frames = frame_indx[in_grp]
# Find the detectors to reduce
# detectors = PypeIt.select_detectors(detnum=self.par['rdx']['detnum'],
# ndet=self.spectrograph.ndet)
detectors = self.spectrograph.select_detectors(
subset=self.par['rdx']['detnum'])
calib_dict[calib_grp] = {}
# Loop on Detectors
for self.det in detectors:
# Instantiate Calibrations class
self.caliBrate = calibrations.Calibrations.get_instance(
self.fitstbl,
self.par['calibrations'],
self.spectrograph,
self.calibrations_path,
qadir=self.qa_path,
reuse_masters=self.reuse_masters,
show=self.show,
user_slits=slittrace.merge_user_slit(
self.par['rdx']['slitspatnum'],
self.par['rdx']['maskIDs']))
# Do it
# TODO: Why isn't set_config part of the Calibrations.__init__ method?
self.caliBrate.set_config(grp_frames[0], self.det,
self.par['calibrations'])
# Allow skipping the run (e.g. parse_calib_id.py script)
if run:
self.caliBrate.run_the_steps()
key = self.caliBrate.master_key_dict['frame']
calib_dict[calib_grp][key] = {}
for step in self.caliBrate.steps:
if step in ['bpm', 'slits', 'wv_calib', 'tilts', 'flats']:
continue
elif step == 'tiltimg': # Annoying kludge
step = 'tilt'
# Prep
raw_files, self.caliBrate.master_key_dict[
step] = self.caliBrate._prep_calibrations(step)
masterframe_name = masterframe.construct_file_name(
buildimage.frame_image_classes[step],
self.caliBrate.master_key_dict[step],
master_dir=self.caliBrate.master_dir)
# Add to dict
if len(raw_files) > 0:
calib_dict[calib_grp][key][step] = {}
calib_dict[calib_grp][key][step][
'master_key'] = self.caliBrate.master_key_dict[
step]
calib_dict[calib_grp][key][step][
'master_name'] = os.path.basename(masterframe_name)
calib_dict[calib_grp][key][step]['raw_files'] = [
os.path.basename(ifile) for ifile in raw_files
]
# Print the results
print(json.dumps(calib_dict, sort_keys=True, indent=4))
# Write
msgs.info('Writing calib file')
calib_file = self.pypeit_file.replace('.pypeit', '.calib_ids')
ltu.savejson(calib_file, calib_dict, overwrite=True, easy_to_read=True)
# Finish
self.print_end_time()
# Return
return calib_dict
def iterative_fitting(spec, tcent, ifit, IDs, llist, disp,
match_toler = 2.0, func = 'legendre', n_first=2, sigrej_first=2.0,
n_final=4, sigrej_final=3.0,
weights=None, plot_fil=None, verbose=False):
""" Routine for iteratively fitting wavelength solutions.
Parameters
----------
spec : ndarray, shape = (nspec,)
arcline spectrum
tcent : ndarray
Centroids in pixels of lines identified in spec
ifit : ndarray
Indices of the lines that will be fit
IDs: ndarray
wavelength IDs of the lines that will be fit (I think?)
llist: dict
Linelist dictionary
disp: float
dispersion
Optional Parameters
-------------------
match_toler: float, default = 3.0
Matching tolerance when searching for new lines. This is the difference in pixels between the wavlength assigned to
an arc line by an iteration of the wavelength solution to the wavelength in the line list.
func: str, default = 'legendre'
Name of function used for the wavelength solution
n_first: int, default = 2
Order of first guess to the wavelength solution.
sigrej_first: float, default = 2.0
Number of sigma for rejection for the first guess to the wavelength solution.
n_final: int, default = 4
Order of the final wavelength solution fit
sigrej_final: float, default = 3.0
Number of sigma for rejection for the final fit to the wavelength solution.
weights: ndarray
Weights to be used?
verbose : bool
If True, print out more information.
plot_fil:
Filename for plotting some QA?
Returns
-------
final_fit: dict
Dictionary containing the full fitting results and the final best guess of the line IDs
"""
#TODO JFH add error checking here to ensure that IDs and ifit have the same size!
if weights is None:
weights = np.ones(tcent.size)
nspec = spec.size
xnspecmin1 = float(nspec-1)
# Setup for fitting
sv_ifit = list(ifit) # Keep the originals
all_ids = -999.*np.ones(len(tcent))
all_idsion = np.array(['UNKNWN']*len(tcent))
all_ids[ifit] = IDs
# Fit
n_order = n_first
flg_continue = True
flg_penultimate = False
fmin, fmax = 0.0, 1.0
# Note the number of parameters is actually n_order and not n_order+1
while flg_continue:
if flg_penultimate:
flg_continue = False
# Fit with rejection
xfit, yfit, wfit = tcent[ifit], all_ids[ifit], weights[ifit]
mask, fit = utils.robust_polyfit(xfit/xnspecmin1, yfit, n_order, function=func, sigma=sigrej_first,
minx=fmin, maxx=fmax, verbose=verbose, weights=wfit)
rms_ang = utils.calc_fit_rms(xfit[mask == 0]/xnspecmin1, yfit[mask == 0], fit, func, minx=fmin, maxx=fmax,
weights=wfit[mask == 0])
rms_pix = rms_ang/disp
if verbose:
msgs.info('n_order = {:d}'.format(n_order) + ': RMS = {:g}'.format(rms_pix))
# Reject but keep originals (until final fit)
ifit = list(ifit[mask == 0]) + sv_ifit
# Find new points (should we allow removal of the originals?)
twave = utils.func_val(fit, tcent/xnspecmin1, func, minx=fmin, maxx=fmax)
for ss, iwave in enumerate(twave):
mn = np.min(np.abs(iwave-llist['wave']))
if mn/disp < match_toler:
imn = np.argmin(np.abs(iwave-llist['wave']))
#if verbose:
# print('Adding {:g} at {:g}'.format(llist['wave'][imn],tcent[ss]))
# Update and append
all_ids[ss] = llist['wave'][imn]
all_idsion[ss] = llist['ion'][imn]
ifit.append(ss)
# Keep unique ones
ifit = np.unique(np.array(ifit, dtype=int))
# Increment order?
if n_order < n_final:
n_order += 1
else:
flg_penultimate = True
# Final fit (originals can now be rejected)
#fmin, fmax = 0., 1.
#xfit, yfit, wfit = tcent[ifit]/(nspec-1), all_ids[ifit], weights[ifit]
xfit, yfit, wfit = tcent[ifit], all_ids[ifit], weights[ifit]
mask, fit = utils.robust_polyfit(xfit/xnspecmin1, yfit, n_order, function=func, sigma=sigrej_final,
minx=fmin, maxx=fmax, verbose=verbose, weights=wfit)#, debug=True)
irej = np.where(mask == 1)[0]
if len(irej) > 0:
xrej = xfit[irej]
yrej = yfit[irej]
if verbose:
for kk, imask in enumerate(irej):
wave = utils.func_val(fit, xrej[kk]/xnspecmin1, func, minx=fmin, maxx=fmax)
msgs.info('Rejecting arc line {:g}; {:g}'.format(yfit[imask], wave))
else:
xrej = []
yrej = []
#xfit = xfit[mask == 0]
#yfit = yfit[mask == 0]
#wfit = wfit[mask == 0]
ions = all_idsion[ifit]
# ions = all_idsion[ifit][mask == 0]
# Final RMS
rms_ang = utils.calc_fit_rms(xfit[mask==0]/xnspecmin1, yfit[mask==0], fit, func,
minx=fmin, maxx=fmax, weights=wfit[mask==0])
# rms_ang = utils.calc_fit_rms(xfit, yfit, fit, func,
# minx=fmin, maxx=fmax, weights=wfit)
rms_pix = rms_ang/disp
# Pack up fit
spec_vec = np.arange(nspec)
wave_soln = utils.func_val(fit,spec_vec/xnspecmin1, func, minx=fmin, maxx=fmax)
cen_wave = utils.func_val(fit, float(nspec)/2/xnspecmin1, func, minx=fmin, maxx=fmax)
cen_wave_min1 = utils.func_val(fit, (float(nspec)/2 - 1.0)/xnspecmin1, func, minx=fmin, maxx=fmax)
cen_disp = cen_wave - cen_wave_min1
final_fit = dict(fitc=fit, function=func, pixel_fit=xfit, wave_fit=yfit, weights=wfit, ions=ions,
fmin=fmin, fmax=fmax, xnorm = xnspecmin1, nspec=nspec, cen_wave = cen_wave, cen_disp = cen_disp,
xrej=xrej, yrej=yrej, mask=(mask == 0), spec=spec, wave_soln = wave_soln, nrej=sigrej_final,
shift=0., tcent=tcent, rms=rms_pix)
# If set to True, this will output a file that can then be included in the tests
saveit = False
if saveit:
from linetools import utils as ltu
jdict = ltu.jsonify(final_fit)
if plot_fil is None:
outname = "temp"
print("You should have set the plot_fil directory to save wavelength fits... using 'temp' as a filename")
else:
outname = plot_fil
ltu.savejson(outname + '.json', jdict, easy_to_read=True, overwrite=True)
print(" Wrote: {:s}".format(outname + '.json'))
# QA
if plot_fil is not None:
autoid.arc_fit_qa(final_fit, plot_fil)
# Return
return final_fit
def semi_brute(spec,
lines,
wv_cen,
disp,
siglev=20.,
min_ampl=300.,
outroot=None,
debug=False,
do_fit=True,
verbose=False,
fit_parm=None,
min_nmatch=0,
lowest_ampl=200.):
"""
Parameters
----------
spec
lines
wv_cen
disp
siglev
min_ampl
outroot
debug
do_fit
verbose
fit_parm
min_nmatch
lowest_ampl
Returns
-------
best_dict : dict
final_fit : dict
"""
# imports
from astropy.table import vstack
from linetools import utils as ltu
from arclines import plots as arcl_plots
# Load line lists
line_lists = arcl_io.load_line_lists(lines)
unknwns = arcl_io.load_unknown_list(lines)
npix = spec.size
# Lines
all_tcent, cut_tcent, icut = arch_utils.arc_lines_from_spec(
spec, min_ampl=min_ampl)
# Best
best_dict = dict(nmatch=0, ibest=-1, bwv=0., min_ampl=min_ampl)
# 3 things to fiddle:
# pix_tol -- higher for fewer lines 1/2
# unknowns -- on for fewer lines off/on
# scoring -- weaken for more lines ??
# Loop on unknowns
for unknown in [False, True]:
if unknown:
tot_list = vstack([line_lists, unknwns])
else:
tot_list = line_lists
wvdata = np.array(tot_list['wave'].data) # Removes mask if any
wvdata.sort()
sav_nmatch = best_dict['nmatch']
# Loop on pix_tol
for pix_tol in [1., 2.]:
# Scan on wavelengths
arch_patt.scan_for_matches(wv_cen,
disp,
npix,
cut_tcent,
wvdata,
best_dict=best_dict,
pix_tol=pix_tol)
# Lower minimum amplitude
ampl = min_ampl
while (best_dict['nmatch'] < min_nmatch):
ampl /= 2.
if ampl < lowest_ampl:
break
all_tcent, cut_tcent, icut = arch_utils.arc_lines_from_spec(
spec, min_ampl=ampl)
arch_patt.scan_for_matches(wv_cen,
disp,
npix,
cut_tcent,
wvdata,
best_dict=best_dict,
pix_tol=pix_tol,
ampl=ampl)
# Save linelist?
if best_dict['nmatch'] > sav_nmatch:
best_dict['line_list'] = tot_list
best_dict['unknown'] = unknown
best_dict['ampl'] = unknown
if best_dict['nmatch'] == 0:
print('---------------------------------------------------')
print('Report:')
print(':: No matches! Could be you input a bad wvcen or disp value')
print('---------------------------------------------------')
return
# Report
print('---------------------------------------------------')
print('Report:')
print(':: Number of lines recovered = {:d}'.format(all_tcent.size))
print(':: Number of lines analyzed = {:d}'.format(cut_tcent.size))
print(':: Number of Perf/Good/Ok matches = {:d}'.format(
best_dict['nmatch']))
print(':: Best central wavelength = {:g}A'.format(best_dict['bwv']))
print(':: Best solution used pix_tol = {}'.format(best_dict['pix_tol']))
print(':: Best solution had unknown = {}'.format(best_dict['unknown']))
print('---------------------------------------------------')
if debug:
match_idx = best_dict['midx']
for kk in match_idx.keys():
uni, counts = np.unique(match_idx[kk]['matches'],
return_counts=True)
print('kk={}, {}, {}, {}'.format(kk, uni, counts, np.sum(counts)))
# Write scores
#out_dict = best_dict['scores']
#jdict = ltu.jsonify(out_dict)
#ltu.savejson(pargs.outroot+'.scores', jdict, easy_to_read=True, overwrite=True)
# Write IDs
if outroot is not None:
out_dict = dict(pix=cut_tcent, IDs=best_dict['IDs'])
jdict = ltu.jsonify(out_dict)
ltu.savejson(outroot + '.json',
jdict,
easy_to_read=True,
overwrite=True)
print("Wrote: {:s}".format(outroot + '.json'))
# Plot
if outroot is not None:
arcl_plots.match_qa(spec, cut_tcent, best_dict['line_list'],
best_dict['IDs'], best_dict['scores'],
outroot + '.pdf')
print("Wrote: {:s}".format(outroot + '.pdf'))
# Fit
final_fit = None
if do_fit:
# Read in Full NIST Tables
full_NIST = arcl_io.load_line_lists(lines, NIST=True)
# KLUDGE!!!!!
keep = full_NIST['wave'] > 8800.
line_lists = vstack([line_lists, full_NIST[keep]])
#
NIST_lines = line_lists['NIST'] > 0
ifit = np.where(best_dict['mask'])[0]
if outroot is not None:
plot_fil = outroot + '_fit.pdf'
else:
plot_fil = None
# Purge UNKNOWNS from ifit
imsk = np.array([True] * len(ifit))
for kk, idwv in enumerate(np.array(best_dict['IDs'])[ifit]):
if np.min(np.abs(line_lists['wave'][NIST_lines] - idwv)) > 0.01:
imsk[kk] = False
ifit = ifit[imsk]
# Allow for weaker lines in the fit
all_tcent, weak_cut_tcent, icut = arch_utils.arc_lines_from_spec(
spec, min_ampl=lowest_ampl)
add_weak = []
for weak in weak_cut_tcent:
if np.min(np.abs(cut_tcent - weak)) > 5.:
add_weak += [weak]
if len(add_weak) > 0:
cut_tcent = np.concatenate([cut_tcent, np.array(add_weak)])
# Fit
final_fit = arch_fit.iterative_fitting(spec,
cut_tcent,
ifit,
np.array(
best_dict['IDs'])[ifit],
line_lists[NIST_lines],
disp,
plot_fil=plot_fil,
verbose=verbose,
aparm=fit_parm)
if plot_fil is not None:
print("Wrote: {:s}".format(plot_fil))
# Return
return best_dict, final_fit
def main(pargs=None):
""" Run
Parameters
----------
args
Returns
-------
"""
import numpy as np
from matplotlib import pyplot as plt
from linetools import utils as ltu
from arclines import io as arcl_io
from arclines.holy import utils as arch_utils
from arclines.holy.grail import general, semi_brute
from arclines.holy import patterns as arch_patt
from arclines.holy import fitting as arch_fit
if pargs.outroot is None:
pargs.outroot = 'tmp_matches'
# Defaults
# Load spectrum
spec = arcl_io.load_spectrum(pargs.spectrum)
if pargs.show_spec:
plt.clf()
ax = plt.gca()
ax.plot(spec)
plt.show()
# Arc lines
lines = pargs.lines.split(',')
# Call brute
if pargs.brute:
best_dict, final_fit = semi_brute(spec,
lines,
pargs.wvcen,
pargs.disp,
min_ampl=pargs.min_ampl,
debug=pargs.debug,
outroot=pargs.outroot,
do_fit=pargs.fit,
verbose=True)
#best_dict, final_fit = grail.semi_brute(spec, lines, wv_cen, disp, siglev=siglev,
# min_ampl=min_ampl, min_nmatch=min_match, outroot=outroot)
else:
best_dict, final_fit = general(spec,
lines,
do_fit=pargs.fit,
verbose=True,
debug=pargs.debug,
min_ampl=pargs.min_ampl,
outroot=pargs.outroot)
if pargs.debug:
pdb.set_trace()
if pargs.fit:
ltu.savejson(pargs.outroot + '_fit.json',
ltu.jsonify(final_fit),
easy_to_read=True,
overwrite=True)
def save_masters(slf, det, mftype='all'):
""" Save Master Frames
Parameters
----------
slf
det : int
mftype : str
'all' -- Save them all
"""
from linetools import utils as ltu
setup = slf.setup
transpose = bool(settings.argflag['trace']['dispersion']['direction'])
# Bias
if (mftype in [
'bias', 'all'
]) and ('bias' + setup
not in settings.argflag['reduce']['masters']['loaded']):
if not isinstance(slf._msbias[det - 1], (basestring)):
arsave.save_master(slf,
slf._msbias[det - 1],
filename=master_name('bias', setup),
frametype='bias')
# Bad Pixel
if (mftype in [
'badpix', 'all'
]) and ('badpix' + setup
not in settings.argflag['reduce']['masters']['loaded']):
arsave.save_master(slf,
slf._bpix[det - 1],
filename=master_name('badpix', setup),
frametype='badpix')
# Trace
if (mftype in [
'trace', 'all'
]) and ('trace' + setup
not in settings.argflag['reduce']['masters']['loaded']):
extensions = [
slf._lordloc[det - 1], slf._rordloc[det - 1], slf._pixcen[det - 1],
slf._pixwid[det - 1], slf._lordpix[det - 1], slf._rordpix[det - 1],
slf._slitpix[det - 1]
]
names = [
'LeftEdges_det', 'RightEdges_det', 'SlitCentre', 'SlitLength',
'LeftEdges_pix', 'RightEdges_pix', 'SlitPixels'
]
arsave.save_master(slf,
slf._mstrace[det - 1],
filename=master_name('trace', setup),
frametype='trace',
extensions=extensions,
names=names)
# Pixel Flat
if (mftype in [
'normpixelflat', 'all'
]) and ('normpixelflat' + setup
not in settings.argflag['reduce']['masters']['loaded']):
arsave.save_master(slf,
slf._mspixelflatnrm[det - 1],
filename=master_name('normpixelflat', setup),
frametype='normpixelflat')
# Pinhole Flat
if (mftype in [
'pinhole', 'all'
]) and ('pinhole' + setup
not in settings.argflag['reduce']['masters']['loaded']):
arsave.save_master(slf,
slf._mspinhole[det - 1],
filename=master_name('pinhole', setup),
frametype='pinhole')
# Arc/Wave
if (mftype in [
'arc', 'all'
]) and ('arc' + setup
not in settings.argflag['reduce']['masters']['loaded']):
arsave.save_master(slf,
slf._msarc[det - 1],
filename=master_name('arc', setup),
frametype='arc',
keywds=dict(transp=transpose))
if (mftype in [
'wave', 'all'
]) and ('wave' + setup
not in settings.argflag['reduce']['masters']['loaded']):
# Wavelength image
arsave.save_master(slf,
slf._mswave[det - 1],
filename=master_name('wave', setup),
frametype='wave')
# Wavelength fit
gddict = ltu.jsonify(slf._wvcalib[det - 1])
json_file = master_name('wv_calib', setup)
if gddict is not None:
ltu.savejson(json_file, gddict, easy_to_read=True, overwrite=True)
else:
msgs.warn("The master wavelength solution has not been saved")
# Tilts
if (mftype in [
'tilts', 'all'
]) and ('tilts' + setup
not in settings.argflag['reduce']['masters']['loaded']):
arsave.save_master(slf,
slf._tilts[det - 1],
filename=master_name('tilts', setup),
frametype='tilts')
# Spatial slit profile
if (mftype in [
'slitprof', 'all'
]) and ('slitprof' + setup
not in settings.argflag['reduce']['masters']['loaded']):
arsave.save_master(slf,
slf._slitprof[det - 1],
filename=master_name('slitprof', setup),
frametype='slit profile')
def general(spec, lines, min_ampl=300.,
outroot=None, debug=False, do_fit=True, verbose=False,
fit_parm=None, lowest_ampl=200.):
"""
Parameters
----------
spec
lines
siglev
min_ampl
outroot
debug
do_fit
verbose
fit_parm
min_nmatch
lowest_ampl
Returns
-------
best_dict : dict
final_fit : dict
"""
# imports
from astropy.table import vstack
from linetools import utils as ltu
from arclines import plots as arcl_plots
# Import the triangles algorithm
from arclines.holy.patterns import triangles
# Load line lists
line_lists = arcl_io.load_line_lists(lines)
unknwns = arcl_io.load_unknown_list(lines)
npix = spec.size
# Lines
all_tcent, cut_tcent, icut = arch_utils.arc_lines_from_spec(spec, min_ampl=min_ampl)
use_tcent = all_tcent.copy()
#use_tcent = cut_tcent.copy() # min_ampl is having not effect at present
# Best
best_dict = dict(nmatch=0, ibest=-1, bwv=0., min_ampl=min_ampl)
ngrid = 1000
# Loop on unknowns
for unknown in [False, True]:
if unknown:
tot_list = vstack([line_lists,unknwns])
else:
tot_list = line_lists
wvdata = np.array(tot_list['wave'].data) # Removes mask if any
wvdata.sort()
sav_nmatch = best_dict['nmatch']
# Loop on pix_tol
for pix_tol in [1.]:#, 2.]:
# Triangle pattern matching
dindex, lindex, wvcen, disps = triangles(use_tcent, wvdata, npix, 5, 10, pix_tol)
# Remove any invalid results
ww = np.where((wvcen > 0.0) & (disps > 0.0))
dindex = dindex[ww[0], :]
lindex = lindex[ww[0], :]
disps = disps[ww]
wvcen = wvcen[ww]
# Setup the grids and histogram
binw = np.linspace(max(np.min(wvcen), np.min(wvdata)), min(np.max(wvcen), np.max(wvdata)), ngrid)
bind = np.linspace(np.min(np.log10(disps)), np.max(np.log10(disps)), ngrid)
histimg, xed, yed = np.histogram2d(wvcen, np.log10(disps), bins=[binw, bind])
histimg = gaussian_filter(histimg, 3)
# Find the best combination of central wavelength and dispersion
bidx = np.unravel_index(np.argmax(histimg), histimg.shape)
debug = False
if debug:
from matplotlib import pyplot as plt
plt.clf()
plt.imshow(histimg[:, ::-1].T, extent=[binw[0], binw[-1], bind[0], bind[-1]], aspect='auto')
plt.axvline(binw[bidx[0]], color='r', linestyle='--')
plt.axhline(bind[bidx[1]], color='r', linestyle='--')
plt.show()
print(histimg[bidx], binw[bidx[0]], 10.0**bind[bidx[1]])
pdb.set_trace()
# Find all good solutions
nsel = 5 # Select all solutions around the best solution within a square of side 2*nsel
wlo = binw[bidx[0] - nsel]
whi = binw[bidx[0] + nsel]
dlo = 10.0 ** bind[bidx[1] - 5*nsel]
dhi = 10.0 ** bind[bidx[1] + 5*nsel]
wgd = np.where((wvcen > wlo) & (wvcen < whi) & (disps > dlo) & (disps < dhi))
dindex = dindex[wgd[0], :].flatten()
lindex = lindex[wgd[0], :].flatten()
# Given this solution, fit for all detlines
arch_patt.solve_triangles(use_tcent, wvdata, dindex, lindex, best_dict)
if best_dict['nmatch'] > sav_nmatch:
best_dict['pix_tol'] = pix_tol
# Save linelist?
if best_dict['nmatch'] > sav_nmatch:
best_dict['bwv'] = binw[bidx[0]]
best_dict['bdisp'] = 10.0**bind[bidx[1]]
best_dict['line_list'] = tot_list.copy()
best_dict['unknown'] = unknown
best_dict['ampl'] = unknown
# Try to pick up some extras by turning off/on unknowns
if best_dict['unknown']:
tot_list = line_lists
else:
tot_list = vstack([line_lists,unknwns])
# Retrieve the wavelengths of the linelist and sort
wvdata = np.array(tot_list['wave'].data) # Removes mask if any
wvdata.sort()
if best_dict['nmatch'] == 0:
print('---------------------------------------------------')
print('Report:')
print(':: No matches! Try another algorithm')
print('---------------------------------------------------')
return
# Report
print('---------------------------------------------------')
print('Report:')
print(':: Number of lines recovered = {:d}'.format(all_tcent.size))
print(':: Number of lines analyzed = {:d}'.format(use_tcent.size))
print(':: Number of acceptable matches = {:d}'.format(best_dict['nmatch']))
print(':: Best central wavelength = {:g}A'.format(best_dict['bwv']))
print(':: Best dispersion = {:g}A/pix'.format(best_dict['bdisp']))
print(':: Best solution used pix_tol = {}'.format(best_dict['pix_tol']))
print(':: Best solution had unknown = {}'.format(best_dict['unknown']))
print('---------------------------------------------------')
# Write IDs
if outroot is not None:
out_dict = dict(pix=use_tcent, IDs=best_dict['IDs'])
jdict = ltu.jsonify(out_dict)
ltu.savejson(outroot+'.json', jdict, easy_to_read=True, overwrite=True)
print("Wrote: {:s}".format(outroot+'.json'))
# Plot
if outroot is not None:
tmp_list = vstack([line_lists, unknwns])
arcl_plots.match_qa(spec, use_tcent, tmp_list,
best_dict['IDs'], best_dict['scores'], outroot+'.pdf')
print("Wrote: {:s}".format(outroot+'.pdf'))
# Fit
final_fit = None
if do_fit:
# Good lines = NIST or OH
good_lines = np.any([line_lists['NIST']>0, line_lists['ion'] == 'OH'], axis=0)
#
ifit = np.where(best_dict['mask'])[0]
if outroot is not None:
plot_fil = outroot+'_fit.pdf'
else:
plot_fil = None
# Purge UNKNOWNS from ifit
imsk = np.array([True]*len(ifit))
for kk, idwv in enumerate(np.array(best_dict['IDs'])[ifit]):
if np.min(np.abs(line_lists['wave'][good_lines]-idwv)) > 0.01:
imsk[kk] = False
ifit = ifit[imsk]
# Allow for weaker lines in the fit
all_tcent, weak_cut_tcent, icut = arch_utils.arc_lines_from_spec(spec, min_ampl=lowest_ampl)
use_weak_tcent = all_tcent.copy()
add_weak = []
for weak in use_weak_tcent:
if np.min(np.abs(use_tcent-weak)) > 5.:
add_weak += [weak]
if len(add_weak) > 0:
use_tcent = np.concatenate([use_tcent, np.array(add_weak)])
# Fit
final_fit = arch_fit.iterative_fitting(spec, use_tcent, ifit,
np.array(best_dict['IDs'])[ifit], line_lists[good_lines],
best_dict['bdisp'], plot_fil=plot_fil, verbose=verbose,
aparm=fit_parm)
if plot_fil is not None:
print("Wrote: {:s}".format(plot_fil))
# Return
return best_dict, final_fit
