def __init__(self,
spectrum,
atmosphere,
telescope,
disperser,
fast_sim=True):
"""Class to simulate cross spectrum.
Parameters
----------
spectrum: Spectrum
Spectrum instance to load main properties before simulation.
atmosphere: Atmosphere
Atmosphere or AtmosphereGrid instance to make the atmospheric simulation.
telescope: TelescopeTransmission
Telescope transmission.
disperser: Grating
Disperser instance.
fast_sim: bool, optional
If True, do a fast simulation without integrating within the wavelength bins (default: True).
Examples
--------
>>> spectrum, telescope, disperser, target = SimulatorInit("./tests/data/reduc_20170530_134_spectrum.fits")
>>> atmosphere = Atmosphere(airmass=1.2, pressure=800, temperature=10)
>>> sim = SpectrumSimulation(spectrum, atmosphere, telescope, disperser, fast_sim=True)
"""
Spectrum.__init__(self)
for k, v in list(spectrum.__dict__.items()):
self.__dict__[k] = copy.copy(v)
self.my_logger = set_logger(self.__class__.__name__)
self.disperser = disperser
self.telescope = telescope
self.atmosphere = atmosphere
self.fast_sim = fast_sim
# save original pixel distances to zero order
# self.disperser.grating_lambda_to_pixel(self.lambdas, x0=self.x0, order=1)
# now reset data
self.lambdas = None
self.lambdas_order2 = None
self.err = None
self.model = lambda x: np.zeros_like(x)
self.model_err = lambda x: np.zeros_like(x)
lbdas_sed = self.target.wavelengths[0]
sub = np.where((lbdas_sed > parameters.LAMBDA_MIN)
& (lbdas_sed < parameters.LAMBDA_MAX))
self.lambdas_step = min(parameters.LAMBDA_STEP, np.min(lbdas_sed[sub]))
def SimulatorInit(filename, fast_load=False):
""" SimulatorInit
Main function to evaluate several spectra
A grid of spectra will be produced for a given target, airmass and pressure
"""
my_logger = set_logger(__name__)
my_logger.info('\n\tStart SIMULATOR initialisation')
# Load data spectrum
spectrum = Spectrum(filename, fast_load=fast_load)
# TELESCOPE TRANSMISSION
# ------------------------
telescope = TelescopeTransmission(spectrum.filter_label)
# DISPERSER TRANSMISSION
# ------------------------
if not isinstance(spectrum.disperser, str):
disperser = spectrum.disperser
else:
disperser = Hologram(spectrum.disperser)
# STAR SPECTRUM
# ------------------------
if not isinstance(spectrum.target, str):
target = spectrum.target
else:
target = Target(spectrum.target)
return spectrum, telescope, disperser, target
def filter_data(file_names): # pragma: no cover
from scipy.stats import median_absolute_deviation
D = []
chi2 = []
dx = []
amplitude = []
regs = []
for name in file_names:
# try:
spectrum = Spectrum(name, fast_load=True)
D.append(spectrum.header["D2CCD"])
dx.append(spectrum.header["PIXSHIFT"])
regs.append(np.log10(spectrum.header["PSF_REG"]))
amplitude.append(np.sum(spectrum.data[300:]))
if "CHI2_FIT" in spectrum.header:
chi2.append(spectrum.header["CHI2_FIT"])
# except:
# print(f"fail to open {name}")
D = np.array(D)
dx = np.array(dx)
regs = np.array(regs)
chi2 = np.array(chi2)
k = np.arange(len(D))
plt.plot(k, amplitude)
plt.show()
plt.plot(k, D)
# plt.plot(k, np.polyval(np.polyfit(k, reg, deg=1), k))
plt.axhline(np.median(D))
plt.axhline(np.median(D) + 3 * median_absolute_deviation(D))
plt.axhline(np.median(D) - 3 * median_absolute_deviation(D))
plt.grid()
plt.title("D2CCD")
plt.show()
filter_indices = np.logical_and(D > np.median(D) - 3 * median_absolute_deviation(D),
D < np.median(D) + 3 * median_absolute_deviation(D))
if len(chi2) > 0:
filter_indices *= np.logical_and(chi2 > np.median(chi2) - 3 * median_absolute_deviation(chi2),
chi2 < np.median(chi2) + 3 * median_absolute_deviation(chi2))
filter_indices *= np.logical_and(dx > np.median(dx) - 3 * median_absolute_deviation(dx),
dx < np.median(dx) + 3 * median_absolute_deviation(dx))
filter_indices *= np.logical_and(regs > np.median(regs) - 3 * median_absolute_deviation(regs),
regs < np.median(regs) + 3 * median_absolute_deviation(regs))
plt.plot(k, D)
plt.title("D2CCD")
plt.plot(k[filter_indices], D[filter_indices], "ko")
plt.show()
plt.plot(k, dx)
plt.title("dx")
plt.plot(k[filter_indices], dx[filter_indices], "ko")
plt.show()
plt.plot(k, regs)
plt.title("regs")
plt.plot(k[filter_indices], regs[filter_indices], "ko")
plt.show()
if len(chi2) > 0:
plt.title("chi2")
plt.plot(k, chi2)
plt.plot(k[filter_indices], chi2[filter_indices], "ko")
plt.show()
return np.array(file_names)[filter_indices]
def test_fitchromaticpsf2d_test(sim_image="./tests/data/sim_20170530_134.fits",
config="./config/ctio.ini"):
image, lambdas_truth, amplitude_truth = load_test(sim_image, config=config)
spectrum = Spectrum(sim_image.replace(".fits", "_spectrum.fits"),
config=config)
plot_residuals(spectrum, lambdas_truth, amplitude_truth)
assert np.isclose(float(image.header['X0_T']),
spectrum.target_pixcoords[0],
atol=0.01)
assert np.isclose(float(image.header['Y0_T']),
spectrum.target_pixcoords[1],
atol=0.01)
assert np.isclose(float(image.header['ROTANGLE']),
spectrum.rotation_angle,
atol=180 / np.pi * 1 / parameters.CCD_IMSIZE)
assert np.isclose(float(image.header['BKGD_LEV']),
np.mean(spectrum.spectrogram_bgd),
atol=2e-3)
assert np.isclose(float(image.header['D2CCD_T']),
spectrum.disperser.D,
atol=0.05)
print(spectrum.chromatic_psf.poly_params[spectrum.lambdas.size + 6:] -
np.array(PSF_POLY_PARAMS_TRUTH)[3:])
print(np.std((amplitude_truth - spectrum.data) / spectrum.err))
def __init__(self,
spectrum,
atmosphere,
telescope,
disperser,
with_background=True,
fast_sim=True):
"""Class to simulate a spectrogram.
Parameters
----------
spectrum: Spectrum
Spectrum instance to load main properties before simulation.
atmosphere: Atmosphere
Atmosphere or AtmosphereGrid instance to make the atmospheric simulation.
telescope: TelescopeTransmission
Telescope transmission.
disperser: Grating
Disperser instance.
"""
Spectrum.__init__(self)
for k, v in list(spectrum.__dict__.items()):
self.__dict__[k] = copy.copy(v)
self.disperser = disperser
self.telescope = telescope
self.atmosphere = atmosphere
self.true_lambdas = None
self.true_spectrum = None
self.lambdas = None
self.err = None
self.model = lambda x, y: np.zeros((x.size, y.size))
self.psf = load_PSF(psf_type=parameters.PSF_TYPE)
self.profile_params = None
self.psf_cube = None
self.psf_cube_order2 = None
self.fix_psf_cube = False
self.fast_sim = fast_sim
self.with_background = with_background
lbdas_sed = self.target.wavelengths[0]
sub = np.where((lbdas_sed > parameters.LAMBDA_MIN)
& (lbdas_sed < parameters.LAMBDA_MAX))
self.lambdas_step = min(parameters.LAMBDA_STEP, np.min(lbdas_sed[sub]))
self.yy, self.xx = np.mgrid[:self.spectrogram_Ny, :self.spectrogram_Nx]
self.pixels = np.asarray([self.xx, self.yy])
def __init__(self,
spectrum,
atmosphere,
telescope,
disperser,
with_background=True,
fast_sim=True,
full_image=False,
with_adr=True):
"""Class to simulate a spectrogram.
Parameters
----------
spectrum: Spectrum
Spectrum instance to load main properties before simulation.
atmosphere: Atmosphere
Atmosphere or AtmosphereGrid instance to make the atmospheric simulation.
telescope: TelescopeTransmission
Telescope transmission.
disperser: Grating
Disperser instance.
with_background: bool, optional
If True, add the background model to the simulated spectrogram (default: True).
fast_sim: bool, optional
If True, perform a fast simulation of the spectrum without integrated the spectrum
in pixel bins (default: True).
full_image: bool, optional
If True, simulate the spectrogram on the full CCD size,
otherwise only the cropped spectrogram (default: False).
with_adr: bool, optional
If True, simulate the spectrogram with ADR effect (default: True).
Examples
--------
>>> spectrum, telescope, disperser, target = SimulatorInit("./tests/data/reduc_20170530_134_spectrum.fits")
>>> atmosphere = Atmosphere(airmass=1.2, pressure=800, temperature=10)
>>> sim = SpectrogramModel(spectrum, atmosphere, telescope, disperser, with_background=True, fast_sim=True)
"""
Spectrum.__init__(self)
for k, v in list(spectrum.__dict__.items()):
self.__dict__[k] = copy.copy(v)
self.disperser = disperser
self.telescope = telescope
self.atmosphere = atmosphere
self.true_lambdas = None
self.true_spectrum = None
self.lambdas = None
self.err = None
self.model = lambda x, y: np.zeros((x.size, y.size))
self.psf = load_PSF(psf_type=parameters.PSF_TYPE)
self.profile_params = None
self.psf_cube = None
self.psf_cube_order2 = None
self.fix_psf_cube = False
self.fix_atm_sim = False
self.atmosphere_sim = None
self.fast_sim = fast_sim
self.with_background = with_background
self.full_image = full_image
self.with_adr = with_adr
if self.full_image:
self.Nx = parameters.CCD_IMSIZE
self.Ny = self.spectrogram_Ny # too long if =parameters.CCD_IMSIZE
self.r0 = self.x0[0] + 1j * self.spectrogram_y0
else:
self.Nx = self.spectrogram_Nx
self.Ny = self.spectrogram_Ny
self.r0 = self.spectrogram_x0 + 1j * self.spectrogram_y0
lbdas_sed = self.target.wavelengths[0]
sub = np.where((lbdas_sed > parameters.LAMBDA_MIN)
& (lbdas_sed < parameters.LAMBDA_MAX))
self.lambdas_step = min(parameters.LAMBDA_STEP, np.min(lbdas_sed[sub]))
self.yy, self.xx = np.mgrid[:self.Ny, :self.Nx]
self.pixels = np.asarray([self.xx, self.yy])
def fullchain_run(sim_image="./tests/data/sim_20170530_134.fits"):
# load test and make image simulation
load_config("./config/ctio.ini")
if not os.path.isfile(sim_image):
make_image()
image = Image(sim_image)
lambdas_truth = np.fromstring(image.header['LBDAS_T'][1:-1], sep=' ')
amplitude_truth = np.fromstring(image.header['AMPLIS_T'][1:-1],
sep=' ',
dtype=float)
parameters.AMPLITUDE_TRUTH = np.copy(amplitude_truth)
parameters.LAMBDA_TRUTH = np.copy(lambdas_truth)
# extractor
tag = os.path.basename(sim_image)
tag = tag.replace('sim_', 'reduc_')
logbook = LogBook(logbook="./ctiofulllogbook_jun2017_v5.csv")
disperser_label, target, xpos, ypos = logbook.search_for_image(tag)
parameters.PSF_POLY_ORDER = PSF_POLY_ORDER
spectrum = Spectractor(sim_image,
"./tests/data",
guess=[xpos, ypos],
target_label=target,
disperser_label=disperser_label)
# spectrum = Spectrum("./tests/data/sim_20170530_134_spectrum.fits")
# spectrum = Spectrum("./tests/data/sim_20170530_176_spectrum.fits")
# tests
residuals = plot_residuals(spectrum, lambdas_truth, amplitude_truth)
spectrum.my_logger.warning(
f"\n\tQuantities to test:"
f"\n\t\tspectrum.header['X0_T']={spectrum.header['X0_T']:.5g} vs {spectrum.x0[0]:.5g}"
f"\n\t\tspectrum.header['Y0_T']={spectrum.header['Y0_T']:.5g} vs {spectrum.x0[1]:.5g}"
f"\n\t\tspectrum.header['ROT_T']={spectrum.header['ROT_T']:.5g} "
f"vs {spectrum.rotation_angle:.5g}"
f"\n\t\tspectrum.header['BKGD_LEV']={spectrum.header['BKGD_LEV']:.5g} "
f"vs {np.mean(spectrum.spectrogram_bgd):.5g}"
f"\n\t\tspectrum.header['D2CCD_T']={spectrum.header['D2CCD_T']:.5g} "
f"vs {spectrum.disperser.D:.5g}"
f"\n\t\tspectrum.header['A2_FIT']={spectrum.header['A2_FIT']:.5g} vs {A2_T:.5g}"
f"\n\t\tspectrum.header['CHI2_FIT']={spectrum.header['CHI2_FIT']:.4g}"
f"\n\t\tspectrum.chromatic_psf.poly_params="
f"{spectrum.chromatic_psf.poly_params[spectrum.chromatic_psf.Nx + 2 * (PSF_POLY_ORDER + 1):-1]}"
f" vs {PSF_POLY_PARAMS_TRUTH[2 * (PSF_POLY_ORDER + 1):-1]}"
f"\n\t\tresiduals wrt truth: mean={np.mean(residuals[100:-100]):.5g}, "
f"std={np.std(residuals[100:-100]):.5g}")
assert np.isclose(float(spectrum.header['X0_T']), spectrum.x0[0], atol=0.2)
assert np.isclose(float(spectrum.header['Y0_T']), spectrum.x0[1], atol=0.5)
assert np.isclose(float(spectrum.header['ROT_T']),
spectrum.rotation_angle,
atol=180 / np.pi * 1 / parameters.CCD_IMSIZE)
assert np.isclose(float(spectrum.header['BKGD_LEV']),
np.mean(spectrum.spectrogram_bgd),
rtol=1e-2)
assert np.isclose(float(spectrum.header['D2CCD_T']),
spectrum.disperser.D,
atol=0.1)
assert float(spectrum.header['CHI2_FIT']) < 0.65
assert np.all(
np.isclose(
spectrum.chromatic_psf.poly_params[spectrum.chromatic_psf.Nx + 2 *
(PSF_POLY_ORDER + 1):-1],
np.array(PSF_POLY_PARAMS_TRUTH)[2 * (PSF_POLY_ORDER + 1):-1],
rtol=0.1,
atol=0.1))
assert np.abs(np.mean(residuals[100:-100])) < 0.25
assert np.std(residuals[100:-100]) < 2
spectrum_file_name = "./tests/data/sim_20170530_134_spectrum.fits"
# spectrum_file_name = "./tests/data/sim_20170530_176_spectrum.fits"
assert os.path.isfile(spectrum_file_name)
spectrum = Spectrum(spectrum_file_name)
atmgrid_filename = sim_image.replace('sim', 'reduc').replace(
'.fits', '_atmsim.fits')
assert os.path.isfile(atmgrid_filename)
def convert_from_fits_to_txt():
prod_name = parameters.PROD_NAME
prod_txt = parameters.PROD_TXT
if os.path.exists(prod_txt) == False:
os.makedirs(prod_txt)
to_convert_list = []
Lsimutxt = glob.glob(prod_txt + "/sim*spectrum.txt")
Lreductxt = glob.glob(prod_txt + "/reduc*spectrum.txt")
Lsimufits = glob.glob(prod_name + "/sim*spectrum.fits")
Lreducfits = glob.glob(prod_name + "/reduc*spectrum.fits")
Ldefaut = glob.glob(prod_name + "/*20170530_201_spectrum*") + glob.glob(
prod_name +
"/*20170530_200_spectrum*") + glob.glob(prod_name +
"/*20170530_205_spectrum*")
Lsimutxt = [i for i in Lsimutxt if i not in Ldefaut]
Lreductxt = [i for i in Lreductxt if i not in Ldefaut]
Lsimufits = [i for i in Lsimufits if i not in Ldefaut]
Lreducfits = [i for i in Lreducfits if i not in Ldefaut]
if len(Lsimutxt) != len(Lsimufits) or len(Lreductxt) != len(Lreducfits):
for file in Lsimufits:
tag = file.split('/')[-1]
fichier = os.path.join(prod_txt, tag.replace('fits', 'txt'))
if fichier not in Lsimutxt:
to_convert_list.append(file)
for file in Lreducfits:
tag = file.split('/')[-1]
fichier = os.path.join(prod_txt, tag.replace('fits', 'txt'))
if fichier not in Lreductxt:
to_convert_list.append(file)
for i in range(len(to_convert_list)):
startest = to_convert_list[i]
s = Spectrum(startest)
airmass = s.header["AIRMASS"]
TARGETX = s.header["TARGETX"]
TARGETY = s.header["TARGETY"]
D2CCD = s.header["D2CCD"]
PIXSHIFT = s.header["PIXSHIFT"]
ROTANGLE = s.header["ROTANGLE"]
psf_transverse = s.chromatic_psf.table['fwhm']
PARANGLE = s.header["PARANGLE"]
PSF_REG = s.header['PSF_REG']
x0 = [TARGETX, TARGETY]
print(to_convert_list[i][:len(to_convert_list[i]) - 5])
disperser = s.disperser
distance = disperser.grating_lambda_to_pixel(s.lambdas,
x0=x0,
order=1)
distance += adr_calib(s.lambdas,
s.adr_params,
parameterss.OBS_LATITUDE,
lambda_ref=s.lambda_ref)
distance -= adr_calib(s.lambdas / 2,
s.adr_params,
parameterss.OBS_LATITUDE,
lambda_ref=s.lambda_ref)
lambdas_order2 = disperser.grating_pixel_to_lambda(distance,
x0=x0,
order=2)
disperseur = s.disperser_label
star = s.header['TARGET']
lambda_obs = s.lambdas
intensite_obs = s.data
intensite_err = s.err
cov = s.cov_matrix
if s.target.wavelengths == []:
print('CALSPEC error')
else:
lambda_reel = s.target.wavelengths[0]
intensite_reel = s.target.spectra[0]
tag = to_convert_list[i].split('/')[-1]
fichier = open(
os.path.join(prod_txt, tag.replace('fits', 'txt')), 'w')
fichier.write('#' + '\t' + star + '\t' + disperseur + '\t' +
str(airmass) + '\t' + str(TARGETX) + '\t' +
str(TARGETY) + '\t' + str(D2CCD) + '\t' +
str(PIXSHIFT) + '\t' + str(ROTANGLE) + '\t' +
str(PARANGLE) + '\t' + str(PSF_REG) + '\n')
for j in range(len(lambda_reel)):
if len(lambda_obs) > j:
if len(psf_transverse) > j:
fichier.write(
str(lambda_reel[j]) + '\t' +
str(intensite_reel[j]) + '\t' +
str(lambda_obs[j]) + '\t' +
str(intensite_obs[j]) + '\t' +
str(intensite_err[j]) + '\t' +
str(lambdas_order2[j]) + '\t' +
str(psf_transverse[j]) + '\n')
else:
fichier.write(
str(lambda_reel[j]) + '\t' +
str(intensite_reel[j]) + '\t' +
str(lambda_obs[j]) + '\t' +
str(intensite_obs[j]) + '\t' +
str(intensite_err[j]) + '\t' +
str(lambdas_order2[j]) + '\n')
else:
fichier.write(
str(lambda_reel[j]) + '\t' +
str(intensite_reel[j]) + '\n')
fichier.close()
np.save(os.path.join(prod_txt, tag.replace('fits', 'npy')),
cov)
return False, Lsimutxt, Lreductxt
else:
print('already done')
return True, Lsimutxt, Lreductxt
def run(self, exp, xpos, ypos, target, outputRoot=None, plotting=True):
# run option kwargs in the original code, seems to ~always be True
atmospheric_lines = True
self.log.info('Starting SPECTRACTOR')
# TODO: rename _makePath _makeOutputPath
if outputRoot is not None: # TODO: remove post Gen3 transition
self._makePath(outputRoot, plotting=plotting) # early in case this fails, as processing is slow
# Upstream loads config file here
# TODO: passing exact centroids seems to be causing a serious
# and non-obvious problem!
# this needs fixing for several reasons, mostly because if we have a
# known good centroid then we want to skip the refitting entirely
xpos = int(np.round(xpos))
ypos = int(np.round(ypos))
filter_label, disperser = self._getFilterAndDisperserFromExp(exp)
image = self.spectractorImageFromLsstExposure(exp, target_label=target, disperser_label=disperser,
filter_label=filter_label)
if self.TRANSPOSE:
xpos, ypos = self.transposeCentroid(xpos, ypos, image)
image.target_guess = (xpos, ypos)
if parameters.DEBUG:
image.plot_image(scale='log10', target_pixcoords=image.target_guess)
self.log.info(f"Pixel value at centroid = {image.data[int(ypos), int(xpos)]}")
# XXX this needs removing or at least dealing with to not always
# just run! ASAP XXX
# if disperser == 'ronchi170lpmm':
# TODO: add something more robust as to whether to flip!
# image, xpos, ypos = self.flipImageLeftRight(image, xpos, ypos)
# self.displayImage(image, centroid=(xpos, ypos))
# Use fast mode
if parameters.CCD_REBIN > 1:
self.log.info(f'Rebinning image with rebin of {parameters.CCD_REBIN}')
# TODO: Fix bug here where the passed parameter isn't used!
image.rebin()
if parameters.DEBUG:
image.plot_image(scale='symlog', target_pixcoords=image.target_guess)
# image turning and target finding - use LSST code instead?
# and if not, at least test how the rotation code compares
# this part of Spectractor is certainly slow at the very least
if True: # TODO: change this to be an option, at least for testing vs LSST
self.log.info('Search for the target in the image...')
_ = find_target(image, image.target_guess) # sets the image.target_pixcoords
turn_image(image) # creates the rotated data, and sets the image.target_pixcoords_rotated
# Rotate the image: several methods
# Find the exact target position in the rotated image:
# several methods - but how are these controlled? MFL
self.log.info('Search for the target in the rotated image...')
_ = find_target(image, image.target_guess, rotated=True)
else:
# code path for if the image is pre-rotated by LSST code
raise NotImplementedError
# Create Spectrum object
spectrum = Spectrum(image=image, order=parameters.SPEC_ORDER) # XXX new in DM-33589 check SPEC_ORDER
self.setAdrParameters(spectrum, exp)
# Subtract background and bad pixels
w_psf1d, bgd_model_func = extract_spectrum_from_image(image, spectrum,
signal_width=parameters.PIXWIDTH_SIGNAL,
ws=(parameters.PIXDIST_BACKGROUND,
parameters.PIXDIST_BACKGROUND
+ parameters.PIXWIDTH_BACKGROUND),
right_edge=parameters.CCD_IMSIZE)
spectrum.atmospheric_lines = atmospheric_lines
# PSF2D deconvolution
if parameters.SPECTRACTOR_DECONVOLUTION_PSF2D:
run_spectrogram_deconvolution_psf2d(spectrum, bgd_model_func=bgd_model_func)
# Calibrate the spectrum
with_adr = True
if parameters.OBS_OBJECT_TYPE != "STAR":
# XXX Check what this is set to, and how
# likely need to be passed through
with_adr = False
calibrate_spectrum(spectrum, with_adr=with_adr)
# not necessarily set during fit but required to be present for astropy
# fits writing to work (required to be in keeping with upstream)
spectrum.data_order2 = np.zeros_like(spectrum.lambdas_order2)
spectrum.err_order2 = np.zeros_like(spectrum.lambdas_order2)
# Full forward model extraction:
# adds transverse ADR and order 2 subtraction
w = None
if parameters.SPECTRACTOR_DECONVOLUTION_FFM:
w = FullForwardModelFitWorkspace(spectrum, verbose=parameters.VERBOSE, plot=True, live_fit=False,
amplitude_priors_method="spectrum")
spectrum = run_ffm_minimisation(w, method="newton", niter=2)
# Save the spectrum
self._ensureFitsHeader(spectrum) # SIMPLE is missing by default
# Plot the spectrum
parameters.DISPLAY = True
if parameters.VERBOSE and parameters.DISPLAY:
spectrum.plot_spectrum(xlim=None)
spectrum.chromatic_psf.table['lambdas'] = spectrum.lambdas
result = Spectraction()
result.spectrum = spectrum
result.image = image
result.w = w
# XXX technically this should be a pipeBase.Struct I think
# change it if it matters
return result
def _readFile(self, path, pytype=None):
return Spectrum(path)