def doone(seed,
sn=100,
nlam=400,
config_name='tests/test.yaml',
doplot=False,
resol=1000):
if config_name != si.fname:
si.config = utils.read_config(config_name)
si.fname = config_name
config = si.config
st = np.random.get_state()
np.random.seed(seed)
# read data
lamcen = 5000.
#lamcen_air = lamcen / (1.0 + 2.735182E-4 + 131.4182 / lamcen**2 +
# 2.76249E8 / lamcen**4)
v0 = np.random.normal(0, 300)
slope = (np.random.uniform(-2, 2))
wresol = (lamcen / resol / 2.35)
lam = np.linspace(4600, 5400, nlam)
#spec0 = (1 - 0.02 * np.exp(-0.5 * ((lam - lamcen1) / w)**2))*lam**slope
teff = np.random.uniform(3000, 12000)
feh = np.random.uniform(-2, 0)
alpha = np.random.uniform(0, 1)
logg = np.random.uniform(0, 5)
c = 299792.458
lam1 = lam / np.sqrt((1 + v0 / c) / (1 - v0 / c))
spec0 = mktemps.getspec(lam1, teff, logg, feh, alpha,
wresol=wresol) * lam**slope
spec0 = spec0 / np.median(spec0) * 10**np.random.uniform(-3, 3)
espec = spec0 / sn
spec = np.random.normal(spec0, espec)
# construct specdata object
specdata = [spec_fit.SpecData('config1', lam, spec, espec)]
options = {'npoly': 10}
paramDict0 = {'logg': 2.5, 'teff': 5000, 'feh': -1, 'alpha': 0.5}
#, 'vsini': 0.0}
# fixParam = ['vsini']
res = vel_fit.process(
specdata,
paramDict0,
# fixParam=fixParam,
config=config,
options=options)
ret = (v0, res['vel'], res['vel_err'])
if doplot:
plt.figure(figsize=(6, 2), dpi=300)
plt.plot(specdata[0].lam, specdata[0].spec, 'k-')
plt.plot(specdata[0].lam, res['yfit'][0], 'r-')
plt.tight_layout()
plt.savefig('plot_accuracy_test.png')
#1/0
np.random.set_state(st)
return ret
def test_fake_grid():
config = utils.read_config(path + '/test.yaml')
# read data
lam = np.linspace(4600, 5400, 800)
v0 = np.random.normal(0, 100)
lam1 = lam / (1 + v0 / 3e5)
resol = 1000.
lamcen = 5000
w = lamcen / resol / 2.35
spec0 = getspec(lam1, 5000, 2, -1, 0.2, wresol=w)
espec = spec0 * 0.01
spec = np.random.normal(spec0, espec)
# construct specdata object
specdata = [spec_fit.SpecData('config1_grid', lam, spec, espec)]
options = {'npoly': 15}
paramDict0 = {
'logg': 2,
'teff': 5000,
'feh': -0.2,
'alpha': 0.2,
'vsini': 0.1
}
fixParam = [] #'vsini']
res = vel_fit.process(specdata,
paramDict0,
fixParam=fixParam,
config=config,
options=options)
print(res['vel'] - v0, res['vel_err'])
print(res['param'])
if True:
plt.figure(figsize=(6, 2), dpi=300)
plt.plot(specdata[0].lam, specdata[0].spec, 'k-')
plt.plot(specdata[0].lam, res['yfit'][0], 'r-')
plt.tight_layout()
plt.savefig(path + '/plot_test_fit_fake_grid.png')
def proc_weave(fnames, fig_prefix, config, threadid, nthreads):
"""
Process One single file with desi spectra
Parameters:
-----------
fname: str
The filename with the spectra to be fitted
ofname: str
The filename where the table with parameters will be stored
fig_prefix: str
The prefix where the figures will be stored
"""
options = {'npoly': 15}
print('Processing', fnames)
fnames = fnames.split(',')
#if not valid_file(fnames[0]):
# return
tab = pyfits.getdata(fnames[0], 'FIBTABLE')
hdr = pyfits.getheader(fnames[0])
#mws = tab['MWS_TARGET']
targetid = tab['TARGID']
brick_name = hdr['OBID'].replace('.', '').replace('/', '').replace('_', '')
#xids = np.nonzero(mws)[0]
#setups = ('b', 'r', 'z')
fluxes = {}
ivars = {}
waves = {}
masks = {}
setups = ('b', 'r')
targcat = tab['TARGCAT']
programs = ['GA_LRhighlat', 'GA_LRdisc']
xids = np.zeros(len(targcat), dtype=bool)
for _p in programs:
xids = xids | (targcat == _p)
xids = np.nonzero(xids)[0]
if len(xids) > 0:
tids = np.linspace(0, nthreads, len(xids), False).astype(int)
assert (tids.max() <= (nthreads - 1))
xids = xids[tids == threadid]
if len(xids) == 0:
return None
arms = [pyfits.getheader(f)['CAMERA'].replace('WEAVE', '') for f in fnames]
if arms == ['RED', 'BLUE'] or arms == ['BLUE', 'RED']:
if arms == ['RED', 'BLUE']:
fnames = fnames[::-1]
else:
raise Exception('No RED/BLUE setups')
for fname, s in zip(fnames, setups):
curarm = {'b': 'BLUE', 'r': 'RED'}[s]
fluxes[s] = pyfits.getdata(fname, '%s_DATA' % curarm)
ivars[s] = pyfits.getdata(fname, '%s_IVAR' % curarm)
masks[s] = (ivars[s] == 0).astype(int)
pix = np.arange(fluxes[s].shape[1])
wc = pywcs.WCS(pyfits.getheader(fname, '%s_DATA' % curarm))
waves[s] = wc.all_pix2world(np.array([pix, pix * 0]).T, 0).T[0] * 1e10
tellurics = (((waves[s] >= 8130) & (waves[s] < 8350)) |
((waves[s] >= 6850) & (waves[s] < 7000)) |
((waves[s] >= 8940) & (waves[s] < 9240)) |
((waves[s] >= 9250) & (waves[s] < 9545)) |
((waves[s] >= 9550) & (waves[s] < 10000)))
#medivar = np.nanmedian(ivars[s], axis=-1)
# inflate the errors in the tellurics 1000 times
ivars[s][:, tellurics] = 1. / 100. / np.maximum(
fluxes[s][:, tellurics], 1)**2 #medivar[:, None]/1000**2
# put the S/N in the telluric region to 1/10.
outdict = pandas.DataFrame()
large_error = 1e9
for curid in xids:
specdata = []
curbrick = brick_name
curtargetid = targetid[curid].replace('"', '')
fig_fname = fig_prefix + '_%s_%s.png' % (curbrick, curtargetid)
sns = {}
chisqs = {}
for s in setups:
spec = fluxes[s][curid]
curivars = ivars[s][curid]
badmask = (curivars <= 0) | (masks[s][curid] > 0)
curivars[badmask] = 1. / large_error**2
espec = 1. / curivars**.5
sns[s] = np.nanmedian(spec / espec)
specdata.append(
spec_fit.SpecData('weave_%s' % s,
waves[s],
spec,
espec,
badmask=badmask))
t1 = time.time()
res = fitter_ccf.fit(specdata, config)
t2 = time.time()
paramDict0 = res['best_par']
fixParam = []
if res['best_vsini'] is not None:
paramDict0['vsini'] = res['best_vsini']
res1 = vel_fit.process(specdata,
paramDict0,
fixParam=fixParam,
config=config,
options=options)
t3 = time.time()
chisq_cont_array = spec_fit.get_chisq_continuum(
specdata, options=options)['chisq_array']
t4 = time.time()
curD = {}
curD['brickname'] = curbrick
curD['target_id'] = curtargetid
curD['vrad'] = res1['vel']
curD['vrad_err'] = res1['vel_err']
curD['logg'] = res1['param']['logg']
curD['teff'] = res1['param']['teff']
curD['alpha'] = res1['param']['alpha']
curD['feh'] = res1['param']['feh']
curD['logg_err'] = res1['param_err']['logg']
curD['teff_err'] = res1['param_err']['teff']
curD['alpha_err'] = res1['param_err']['alpha']
curD['feh_err'] = res1['param_err']['feh']
curD['chisq_tot'] = sum(res1['chisq_array'])
for i, s in enumerate(setups):
curD['chisq_%s' % s] = res1['chisq_array'][i]
curD['chisq_c_%s' % s] = float(chisq_cont_array[i])
curD['sn_%s' % (s, )] = sns[s]
curD['vsini'] = res1['vsini']
outdict = outdict.append(curD, True)
title = 'logg=%.1f teff=%.1f [Fe/H]=%.1f [alpha/Fe]=%.1f Vrad=%.1f+/-%.1f' % (
res1['param']['logg'], res1['param']['teff'], res1['param']['feh'],
res1['param']['alpha'], res1['vel'], res1['vel_err'])
make_plot(specdata, res1, title, fig_fname)
outtab = atpy.Table.from_pandas(outdict)
return outtab
def get_specdata(waves, fluxes, ivars, masks, seqid, setups):
""" Return the list of SpecDatas for one single object
Parameters
----------
waves: ndarray
1d wavelength array
fluxes: ndarray
2d flux array
ivars: ndarray
2d array of inverse variances
masks: ndarray
2d array of masks
seqid: int
Which spectral row to extract
setups: list
List of configurations
Returns
-------
ret: list
List of specfit.SpecData objects or None if failed
"""
large_error = 1000
sds = []
minerr_frac = 0.3 # if the error is smaller than this times median error
# clamp the uncertainty
for s in setups:
spec = fluxes[s][seqid] * 1
curivars = ivars[s][seqid] * 1
medspec = np.nanmedian(spec)
if medspec == 0:
medspec = np.nanmedian(spec[spec > 0])
if not np.isfinite(medspec):
medspec = np.nanmedian(np.abs(spec))
if not np.isfinite(medspec) or medspec == 0:
# bail out the spectrum is insane
# TODO make the logic clearer
return None
baddat = ~np.isfinite(spec + curivars)
dicroicmask = (waves[s] > 4300) & (waves[s] < 4450)
badmask = (masks[s][seqid] > 0)
baderr = curivars <= 0
badall = baddat | badmask | baderr | dicroicmask
curivars[badall] = 1. / medspec**2 / large_error**2
spec[badall] = medspec
espec = 1. / curivars**.5
if badall.all():
logging.warning('The whole spectrum was masked...')
else:
goodespec = espec[~badall]
goodespec_thresh = np.median(goodespec) * minerr_frac
replace_idx = (espec < goodespec_thresh) & (~badall)
if replace_idx.sum() / (~badall).sum() > .1:
logging.warning(
'More than 10% of spectra had the uncertainty clamped')
# logging.debug("Clamped error on %d pixels" % (replace_idx.sum()))
espec[replace_idx] = goodespec_thresh
sd = spec_fit.SpecData('desi_%s' % s,
waves[s],
spec,
espec,
badmask=badall)
sds.append(sd)
return sds
def test_fits():
config = utils.read_config(path + '/config.yaml')
# read data
dat = pyfits.getdata(path + '/data/spec-0266-51602-0031.fits')
err = dat['ivar']
err = 1. / err**.5
err[~np.isfinite(err)] = 1e40
# construct specdata object
specdata = [
spec_fit.SpecData('sdss1', 10**dat['loglam'], dat['flux'], err)
]
rot_params = None
resols_params = None
params_list = [[4000, 3, -1, 0], [5000, 3, -1, 0], [6000, 2, -2, 0],
[5500, 5, 0, 0]]
vel_grid = np.linspace(-600, 600, 1000)
options = {'npoly': 10}
t1 = time.time()
res = spec_fit.find_best(specdata,
vel_grid,
params_list,
rot_params,
resols_params,
options=options,
config=config)
t2 = time.time()
bestv, bestpar, bestchi, vel_err = [
res[_] for _ in ['best_vel', 'best_param', 'best_chi', 'vel_err']
]
assert (np.abs(bestv - 15) < 15)
param0 = vel_fit.firstguess(specdata, options=options, config=config)
resfull = vel_fit.process(specdata,
param0,
resols_params,
options=options,
config=config)
chisquare = np.mean(
((specdata[0].spec - resfull['yfit'][0]) / specdata[0].espec)**2)
assert (chisquare < 1.2)
assert (np.abs(resfull['vel'] - 6) < 10)
rot_params = (300, )
plt.clf()
ret = spec_fit.get_chisq(specdata,
bestv,
bestpar,
rot_params,
resols_params,
options=options,
config=config,
full_output=True)
plt.plot(specdata[0].lam, specdata[0].spec, 'k')
plt.plot(specdata[0].lam, ret['models'][0], 'r')
plt.savefig(path + '/plot_sdss_test1.png')
# Test the fit with the resolution matrix
rot_params = None
resol_mat = spec_fit.construct_resol_mat(specdata[0].lam, 50)
resols_params = {'sdss1': resol_mat}
ret = spec_fit.get_chisq(specdata,
bestv,
bestpar,
rot_params,
resol_params=resols_params,
options=options,
config=config,
full_output=True)
plt.clf()
plt.plot(specdata[0].lam, specdata[0].spec, 'k')
plt.plot(specdata[0].lam, ret['models'][0], 'r')
plt.savefig(path + '/plot_sdss_test2.png')
resol_mat = spec_fit.construct_resol_mat(specdata[0].lam, 50)
specdata = [
spec_fit.SpecData('sdss1',
10**dat['loglam'],
dat['flux'],
err,
resolution=resol_mat)
]
ret = spec_fit.get_chisq(specdata,
bestv,
bestpar,
rot_params,
options=options,
config=config,
full_output=True)
plt.clf()
plt.plot(specdata[0].lam, specdata[0].spec, 'k')
plt.plot(specdata[0].lam, ret['models'][0], 'r')
plt.savefig(path + '/plot_sdss_test3.png')
ret = spec_fit.get_chisq_continuum(specdata, options=options)
def test_fit():
config = utils.read_config(path + '/config.yaml')
# read data
dat = pyfits.getdata(path + '/data/spec-0266-51602-0031.fits')
err = dat['ivar']
err = 1. / err**.5
err[~np.isfinite(err)] = 1e40
# construct specdata object
specdata = [
spec_fit.SpecData('sdss1', 10**dat['loglam'], dat['flux'], err)
]
options = {'npoly': 15}
paramDict0 = {
'logg': 2,
'teff': 5000,
'feh': -1,
'alpha': 0.2,
'vsini': 19
}
fixParam = ['vsini']
paramDict0 = {
'logg': 2,
'teff': 5000,
'feh': -1,
'alpha': 0.2,
'vsini': 19
}
fixParam = ['vsini']
# fit with fixed vssini
res = vel_fit.process(specdata,
paramDict0,
fixParam=fixParam,
config=config,
options=options)
paramDict0 = {
'logg': 2,
'teff': 5000,
'feh': -1,
'alpha': 0.2,
'vsini': 19
}
# fit witout fixin
fixParam = []
res = vel_fit.process(specdata,
paramDict0,
fixParam=fixParam,
config=config,
options=options)
options = {'npoly': 15}
# first guess fit
xres0 = vel_fit.firstguess(specdata, config=config)
# ccf
res = fitter_ccf.fit(specdata, config)
paramDict0 = res['best_par']
fixParam = []
if res['best_vsini'] is not None:
paramDict0['vsini'] = res['best_vsini']
res1 = vel_fit.process(specdata,
paramDict0,
fixParam=fixParam,
config=config,
options=options)
print(res1)
plt.figure(figsize=(6, 2), dpi=300)
plt.plot(specdata[0].lam, specdata[0].spec, 'k-')
plt.plot(specdata[0].lam, res1['yfit'][0], 'r-')
plt.tight_layout()
plt.savefig(path + '/plot_test_fit_sdss.png')
# test priors
res2 = vel_fit.process(specdata,
paramDict0,
fixParam=fixParam,
config=config,
options=options,
priors={'teff': (9000, 50)})
# test rbf
options['rbf_continuum'] = False
res2 = vel_fit.process(specdata,
paramDict0,
fixParam=fixParam,
config=config,
options=options)
args.snr)]['spectrum_continuum_normalised']
# Replace errors which are nans with a large value, otherwise they cause numerical failures in the RV code
observed.value_errors[np.isnan(observed.value_errors)] = 1000.
# Resample it onto a logarithmic raster of fixed step
resampler = SpectrumResampler(observed)
# Loop over each arm of this 4MOST mode in turn, populating a list of the observed spectra
spectral_data = []
for arm in arm_rasters[mode_lower]:
observed_arm = resampler.onto_raster(arm['raster'])
spectral_data.append(
spec_fit.SpecData(name=arm['name'],
lam=arm['raster'],
spec=observed_arm.values,
espec=observed_arm.value_errors,
badmask=None))
# Debugging
np.savetxt(
"/tmp/debug_observed_{}.dat".format(run_id),
np.transpose([
observed.wavelengths, observed.values,
observed.value_errors
]))
for arm in arm_rasters[mode_lower]:
np.savetxt(
"/tmp/debug_{}_{}.dat".format(arm['name'], run_id),
np.transpose([