def test_extractor_ctio_planetary_nebula():
file_names = ['tests/data/reduc_20170605_028.fits']
output_directory = "./outputs"
logbook = LogBook(logbook='./ctiofulllogbook_jun2017_v5.csv')
load_config("./config/ctio.ini")
parameters.VERBOSE = True
parameters.DEBUG = True
parameters.CCD_REBIN = 1 # do not work with other values
parameters.LAMBDA_MIN = 450
parameters.LAMBDA_MAX = 1000
for file_name in file_names:
tag = file_name.split('/')[-1]
disperser_label, target_label, xpos, ypos = logbook.search_for_image(
tag)
if target_label is None or xpos is None or ypos is None:
continue
spectrum = Spectractor(file_name,
output_directory,
target_label, [xpos, ypos],
disperser_label,
atmospheric_lines=True)
assert spectrum.data is not None
spectrum.my_logger.warning(
f"\n\tQuantities to test:"
f"\n\t\tspectrum.lambdas[0]={spectrum.lambdas[0]}"
f"\n\t\tspectrum.lambdas[-1]={spectrum.lambdas[-1]}"
f"\n\t\tspectrum.x0={spectrum.x0}"
f"\n\t\tspectrum.spectrogram_x0={spectrum.spectrogram_x0}"
f"\n\t\tspectrum total flux={np.sum(spectrum.data) * parameters.CCD_REBIN ** 2}"
f"\n\t\tnp.mean(spectrum.chromatic_psf.table['gamma']="
f"{np.mean(spectrum.chromatic_psf.table['gamma'])}")
if parameters.SPECTRACTOR_DECONVOLUTION_PSF2D or parameters.SPECTRACTOR_DECONVOLUTION_FFM:
assert np.isclose(spectrum.lambdas[0], 449, atol=1)
assert np.isclose(spectrum.lambdas[-1], 996.5, atol=1)
else:
assert np.isclose(spectrum.lambdas[0], 443, atol=1)
assert np.isclose(spectrum.lambdas[-1], 981, atol=1)
assert np.isclose(spectrum.spectrogram_x0, -368, atol=1)
assert np.sum(
spectrum.data
) * parameters.CCD_REBIN**2 > 1e-11 / parameters.CCD_REBIN
assert np.isclose(spectrum.x0[0] * parameters.CCD_REBIN,
816.75,
atol=0.5 * parameters.CCD_REBIN)
assert np.isclose(spectrum.x0[1] * parameters.CCD_REBIN,
587.67,
atol=1 * parameters.CCD_REBIN)
assert 1 < np.mean(
spectrum.chromatic_psf.table['gamma']) * parameters.CCD_REBIN < 2.5
assert os.path.isfile(
os.path.join(output_directory,
tag.replace('.fits', '_spectrum.fits'))) is True
assert os.path.isfile(
os.path.join(output_directory,
tag.replace('.fits', '_spectrogram.fits'))) is True
assert os.path.isfile(
os.path.join(output_directory, tag.replace('.fits',
'_lines.csv'))) is True
def test_extractor_ctio():
file_names = ['tests/data/reduc_20170530_134.fits']
output_directory = "./outputs"
logbook = LogBook(logbook='./ctiofulllogbook_jun2017_v5.csv')
load_config("./config/ctio.ini")
parameters.VERBOSE = True
parameters.DEBUG = True
parameters.CCD_REBIN = 1
apply_rebinning_to_parameters()
for file_name in file_names:
tag = file_name.split('/')[-1].replace("sim", "reduc")
disperser_label, target_label, xpos, ypos = logbook.search_for_image(
tag)
if target_label is None or xpos is None or ypos is None:
continue
spectrum = Spectractor(file_name,
output_directory,
target_label, [xpos, ypos],
disperser_label,
atmospheric_lines=True)
assert spectrum.data is not None
spectrum.my_logger.warning(
f"\n\tQuantities to test:"
f"\n\t\tspectrum.lambdas[0]={spectrum.lambdas[0]}"
f"\n\t\tspectrum.lambdas[-1]={spectrum.lambdas[-1]}"
f"\n\t\tspectrum.x0={spectrum.x0}"
f"\n\t\tspectrum.spectrogram_x0={spectrum.spectrogram_x0}"
f"\n\t\tspectrum total flux={np.sum(spectrum.data) * parameters.CCD_REBIN ** 2}"
f"\n\t\tnp.mean(spectrum.chromatic_psf.table['gamma']="
f"{np.mean(spectrum.chromatic_psf.table['gamma'])}")
assert np.sum(
spectrum.data
) * parameters.CCD_REBIN**2 > 2e-11 / parameters.CCD_REBIN
if parameters.CCD_REBIN == 1:
if parameters.SPECTRACTOR_DECONVOLUTION_PSF2D or parameters.SPECTRACTOR_DECONVOLUTION_FFM:
assert np.isclose(spectrum.lambdas[0], 343, atol=1)
assert np.isclose(spectrum.lambdas[-1], 1084.0, atol=1)
else:
assert np.isclose(spectrum.lambdas[0], 347, atol=1)
assert np.isclose(spectrum.lambdas[-1], 1085.0, atol=1)
assert np.isclose(spectrum.spectrogram_x0, -280, atol=1)
assert np.isclose(spectrum.x0[0] * parameters.CCD_REBIN,
743.6651370068676,
atol=0.5 * parameters.CCD_REBIN)
assert np.isclose(spectrum.x0[1] * parameters.CCD_REBIN,
683.0577836601408,
atol=1 * parameters.CCD_REBIN)
assert 2 < np.mean(
spectrum.chromatic_psf.table['gamma']) * parameters.CCD_REBIN < 3.5
assert os.path.isfile(
os.path.join(output_directory,
tag.replace('.fits', '_spectrum.fits'))) is True
assert os.path.isfile(
os.path.join(output_directory,
tag.replace('.fits', '_spectrogram.fits'))) is True
assert os.path.isfile(
os.path.join(output_directory, tag.replace('.fits',
'_lines.csv'))) is True
def extractor_auxtel():
file_names = [
'tests/data/calexp_2020031500162-EMPTY_ronchi90lpmm-det000.fits'
] # image 1
# file_names = ['tests/data/calexp_2020031200313-EMPTY_ronchi90lpmm-det000.fits'] # image 2
# file_names = ['tests/data/calexp_2020022100767-EMPTY_ronchi90lpmm-det000.fits'] # image 3
# file_names = ['tests/data//calexp_2020021800154-EMPTY_ronchi90lpmm-det000.fits'] # image 4
file_names = [
'tests/data/Cor_holo4_003_empty_HD60753_2021-02-18_585.fits'
] # ronchi170lpmm
# tests/data/auxtel_first_light-1.fits']
# logbook = LogBook(logbook='./ctiofulllogbook_jun2017_v5.csv')
parameters.VERBOSE = True
parameters.DEBUG = True
#xpos = 1600
#ypos = 2293
#target_label = "HD107696"
for config in ['./config/auxtel.ini']: #'./config/auxtel_quicklook.ini',
for file_name in file_names:
# tag = file_name.split('/')[-1]
# disperser_label, target, xpos, ypos = logbook.search_for_image(tag)
#spectrum = Spectractor(file_name, './outputs/', target_label=target_label, guess=[xpos, ypos],
# config=config, atmospheric_lines=True)
spectrum = Spectractor(file_name,
'./outputs/',
target_label="",
guess=None,
config=config,
atmospheric_lines=True)
assert spectrum.data is not None
assert np.sum(spectrum.data) > 1e-14
def test_fitchromaticpsf2d_run(sim_image="./tests/data/sim_20170530_134.fits",
config="./config/ctio.ini"):
load_test(sim_image, config=config)
tag = sim_image.split('/')[-1]
tag = tag.replace('sim_', 'reduc_')
logbook = LogBook(logbook="./ctiofulllogbook_jun2017_v5.csv")
disperser_label, target, xpos, ypos = logbook.search_for_image(tag)
spectrum = Spectractor(sim_image,
"./tests/data",
guess=[xpos, ypos],
target_label=target,
disperser_label=disperser_label,
config="./config/ctio.ini")
return spectrum
def run_spectractor(file_name: str, output_directory: str,
position: (int, int), target: str, data: bytes) -> str:
"""
Run the main Spectrator methods. This includes: load of the data,
power-spectrum computation, and the fit. The spectrum is dumped on
the disk (won't work in a pure distributed environment).
Parameters
----------
file_name : String
Name of the file to load (full path)
output_directory : String
Name of the output_directory when power-spectra will be written
position : List of two Int
Position of the target (xpos, ypos) as given by the logbook search.
target : String
Name of the target
data : bytes
Output of sc.binaryFiles reading a FITS file, that is
the FITS file in binary to be given to the
hdu.HDUList.fromstring method.
Returns
----------
file_name: String
The name of the processed file.
"""
try:
Spectractor(file_name,
output_directory,
position,
target,
databinary=data)
except ValueError:
file_name = file_name + "_ValueError_BAD"
except RemoteServiceError:
file_name = file_name + "_RemoteServiceError_BAD"
except IndexError:
file_name = file_name + "_IndexError_BAD"
return file_name
load_config(args.config)
logbook = LogBook(logbook=args.logbook)
for file_name in file_names:
disperser_label = args.disperser_label
if parameters.OBS_NAME == "CTIO":
tag = file_name.split('/')[-1]
tag = tag.replace('sim_', 'reduc_')
disperser_label, target_label, xpos, ypos = logbook.search_for_image(
tag)
guess = [xpos, ypos]
if target_label is None or xpos is None or ypos is None:
continue
else:
guess = None
if args.target_xy != "0,0":
xpos, ypos = args.target_xy.split(",")
xpos = float(xpos)
ypos = float(ypos)
guess = [xpos, ypos]
target_label = args.target_label
# if target_label == "" or (xpos == 0 and ypos == 0):
# sys.exit("Options --xy and --target must be used together, one of these seems not set.")
Spectractor(file_name,
args.output_directory,
target_label=target_label,
guess=guess,
disperser_label=disperser_label,
config=args.config)
file_names = glob.glob(os.path.join(args.input, "*/*.fits"))
logbook = LogBook(logbook=args.csv)
rank = MPI.COMM_WORLD.rank
size = MPI.COMM_WORLD.size
if rank == 0:
print("{} files to process".format(len(file_names)))
print("{} processors used".format(size))
for position in range(rank, len(file_names), size):
file_name = file_names[position]
opt = logbook.search_for_image(file_name.split('/')[-1])
target = opt[0]
xpos = opt[1]
ypos = opt[2]
if target is None or xpos is None or ypos is None:
continue
try:
Spectractor(file_name, args.output_directory, [xpos, ypos], target)
except ValueError:
file_name = file_name + "_ValueError_BAD"
print("[{}] {} failed".format(rank, file_name))
except RemoteServiceError:
file_name = file_name + "_RemoteServiceError_BAD"
print("[{}] {} failed".format(rank, file_name))
except IndexError:
file_name = file_name + "_IndexError_BAD"
print("[{}] {} failed".format(rank, file_name))
def fullchain_run(sim_image="./tests/data/sim_20170530_134.fits"):
# load test and make image simulation
load_config("./config/ctio.ini")
if not os.path.isfile(sim_image):
make_image()
image = Image(sim_image)
lambdas_truth = np.fromstring(image.header['LBDAS_T'][1:-1], sep=' ')
amplitude_truth = np.fromstring(image.header['AMPLIS_T'][1:-1],
sep=' ',
dtype=float)
parameters.AMPLITUDE_TRUTH = np.copy(amplitude_truth)
parameters.LAMBDA_TRUTH = np.copy(lambdas_truth)
# extractor
tag = os.path.basename(sim_image)
tag = tag.replace('sim_', 'reduc_')
logbook = LogBook(logbook="./ctiofulllogbook_jun2017_v5.csv")
disperser_label, target, xpos, ypos = logbook.search_for_image(tag)
parameters.PSF_POLY_ORDER = PSF_POLY_ORDER
spectrum = Spectractor(sim_image,
"./tests/data",
guess=[xpos, ypos],
target_label=target,
disperser_label=disperser_label)
# spectrum = Spectrum("./tests/data/sim_20170530_134_spectrum.fits")
# spectrum = Spectrum("./tests/data/sim_20170530_176_spectrum.fits")
# tests
residuals = plot_residuals(spectrum, lambdas_truth, amplitude_truth)
spectrum.my_logger.warning(
f"\n\tQuantities to test:"
f"\n\t\tspectrum.header['X0_T']={spectrum.header['X0_T']:.5g} vs {spectrum.x0[0]:.5g}"
f"\n\t\tspectrum.header['Y0_T']={spectrum.header['Y0_T']:.5g} vs {spectrum.x0[1]:.5g}"
f"\n\t\tspectrum.header['ROT_T']={spectrum.header['ROT_T']:.5g} "
f"vs {spectrum.rotation_angle:.5g}"
f"\n\t\tspectrum.header['BKGD_LEV']={spectrum.header['BKGD_LEV']:.5g} "
f"vs {np.mean(spectrum.spectrogram_bgd):.5g}"
f"\n\t\tspectrum.header['D2CCD_T']={spectrum.header['D2CCD_T']:.5g} "
f"vs {spectrum.disperser.D:.5g}"
f"\n\t\tspectrum.header['A2_FIT']={spectrum.header['A2_FIT']:.5g} vs {A2_T:.5g}"
f"\n\t\tspectrum.header['CHI2_FIT']={spectrum.header['CHI2_FIT']:.4g}"
f"\n\t\tspectrum.chromatic_psf.poly_params="
f"{spectrum.chromatic_psf.poly_params[spectrum.chromatic_psf.Nx + 2 * (PSF_POLY_ORDER + 1):-1]}"
f" vs {PSF_POLY_PARAMS_TRUTH[2 * (PSF_POLY_ORDER + 1):-1]}"
f"\n\t\tresiduals wrt truth: mean={np.mean(residuals[100:-100]):.5g}, "
f"std={np.std(residuals[100:-100]):.5g}")
assert np.isclose(float(spectrum.header['X0_T']), spectrum.x0[0], atol=0.2)
assert np.isclose(float(spectrum.header['Y0_T']), spectrum.x0[1], atol=0.5)
assert np.isclose(float(spectrum.header['ROT_T']),
spectrum.rotation_angle,
atol=180 / np.pi * 1 / parameters.CCD_IMSIZE)
assert np.isclose(float(spectrum.header['BKGD_LEV']),
np.mean(spectrum.spectrogram_bgd),
rtol=1e-2)
assert np.isclose(float(spectrum.header['D2CCD_T']),
spectrum.disperser.D,
atol=0.1)
assert float(spectrum.header['CHI2_FIT']) < 0.65
assert np.all(
np.isclose(
spectrum.chromatic_psf.poly_params[spectrum.chromatic_psf.Nx + 2 *
(PSF_POLY_ORDER + 1):-1],
np.array(PSF_POLY_PARAMS_TRUTH)[2 * (PSF_POLY_ORDER + 1):-1],
rtol=0.1,
atol=0.1))
assert np.abs(np.mean(residuals[100:-100])) < 0.25
assert np.std(residuals[100:-100]) < 2
spectrum_file_name = "./tests/data/sim_20170530_134_spectrum.fits"
# spectrum_file_name = "./tests/data/sim_20170530_176_spectrum.fits"
assert os.path.isfile(spectrum_file_name)
spectrum = Spectrum(spectrum_file_name)
atmgrid_filename = sim_image.replace('sim', 'reduc').replace(
'.fits', '_atmsim.fits')
assert os.path.isfile(atmgrid_filename)
parameters.DEBUG = True
parameters.VERBOSE = True
file_names = args.input
load_config(args.config)
logbook = LogBook(logbook=args.logbook)
for file_name in file_names:
tag = file_name.split('/')[-1]
disperser_label, target, xpos, ypos = logbook.search_for_image(tag)
if target is None or xpos is None or ypos is None:
continue
spectrum_file_name = args.output_directory + '/' + tag.replace(
'.fits', '_spectrum.fits')
atmgrid = AtmosphereGrid(file_name)
SpectrumSimulatorSimGrid(spectrum_file_name, args.output_directory)
image = ImageSim(file_name,
spectrum_file_name,
args.output_directory,
A1=1,
A2=1,
pwv=5,
ozone=300,
aerosols=0.03,
psf_poly_params=None,
with_stars=True)
sim_file_name = args.output_directory + tag.replace('reduc_', 'sim_')
Spectractor(sim_file_name, args.output_directory, target, [xpos, ypos],
disperser_label, args.config)