def test_extractor_ctio_planetary_nebula():
file_names = ['tests/data/reduc_20170605_028.fits']
output_directory = "./outputs"
logbook = LogBook(logbook='./ctiofulllogbook_jun2017_v5.csv')
load_config("./config/ctio.ini")
parameters.VERBOSE = True
parameters.DEBUG = True
parameters.CCD_REBIN = 1 # do not work with other values
parameters.LAMBDA_MIN = 450
parameters.LAMBDA_MAX = 1000
for file_name in file_names:
tag = file_name.split('/')[-1]
disperser_label, target_label, xpos, ypos = logbook.search_for_image(
tag)
if target_label is None or xpos is None or ypos is None:
continue
spectrum = Spectractor(file_name,
output_directory,
target_label, [xpos, ypos],
disperser_label,
atmospheric_lines=True)
assert spectrum.data is not None
spectrum.my_logger.warning(
f"\n\tQuantities to test:"
f"\n\t\tspectrum.lambdas[0]={spectrum.lambdas[0]}"
f"\n\t\tspectrum.lambdas[-1]={spectrum.lambdas[-1]}"
f"\n\t\tspectrum.x0={spectrum.x0}"
f"\n\t\tspectrum.spectrogram_x0={spectrum.spectrogram_x0}"
f"\n\t\tspectrum total flux={np.sum(spectrum.data) * parameters.CCD_REBIN ** 2}"
f"\n\t\tnp.mean(spectrum.chromatic_psf.table['gamma']="
f"{np.mean(spectrum.chromatic_psf.table['gamma'])}")
if parameters.SPECTRACTOR_DECONVOLUTION_PSF2D or parameters.SPECTRACTOR_DECONVOLUTION_FFM:
assert np.isclose(spectrum.lambdas[0], 449, atol=1)
assert np.isclose(spectrum.lambdas[-1], 996.5, atol=1)
else:
assert np.isclose(spectrum.lambdas[0], 443, atol=1)
assert np.isclose(spectrum.lambdas[-1], 981, atol=1)
assert np.isclose(spectrum.spectrogram_x0, -368, atol=1)
assert np.sum(
spectrum.data
) * parameters.CCD_REBIN**2 > 1e-11 / parameters.CCD_REBIN
assert np.isclose(spectrum.x0[0] * parameters.CCD_REBIN,
816.75,
atol=0.5 * parameters.CCD_REBIN)
assert np.isclose(spectrum.x0[1] * parameters.CCD_REBIN,
587.67,
atol=1 * parameters.CCD_REBIN)
assert 1 < np.mean(
spectrum.chromatic_psf.table['gamma']) * parameters.CCD_REBIN < 2.5
assert os.path.isfile(
os.path.join(output_directory,
tag.replace('.fits', '_spectrum.fits'))) is True
assert os.path.isfile(
os.path.join(output_directory,
tag.replace('.fits', '_spectrogram.fits'))) is True
assert os.path.isfile(
os.path.join(output_directory, tag.replace('.fits',
'_lines.csv'))) is True
def test_extractor_ctio():
file_names = ['tests/data/reduc_20170530_134.fits']
output_directory = "./outputs"
logbook = LogBook(logbook='./ctiofulllogbook_jun2017_v5.csv')
load_config("./config/ctio.ini")
parameters.VERBOSE = True
parameters.DEBUG = True
parameters.CCD_REBIN = 1
apply_rebinning_to_parameters()
for file_name in file_names:
tag = file_name.split('/')[-1].replace("sim", "reduc")
disperser_label, target_label, xpos, ypos = logbook.search_for_image(
tag)
if target_label is None or xpos is None or ypos is None:
continue
spectrum = Spectractor(file_name,
output_directory,
target_label, [xpos, ypos],
disperser_label,
atmospheric_lines=True)
assert spectrum.data is not None
spectrum.my_logger.warning(
f"\n\tQuantities to test:"
f"\n\t\tspectrum.lambdas[0]={spectrum.lambdas[0]}"
f"\n\t\tspectrum.lambdas[-1]={spectrum.lambdas[-1]}"
f"\n\t\tspectrum.x0={spectrum.x0}"
f"\n\t\tspectrum.spectrogram_x0={spectrum.spectrogram_x0}"
f"\n\t\tspectrum total flux={np.sum(spectrum.data) * parameters.CCD_REBIN ** 2}"
f"\n\t\tnp.mean(spectrum.chromatic_psf.table['gamma']="
f"{np.mean(spectrum.chromatic_psf.table['gamma'])}")
assert np.sum(
spectrum.data
) * parameters.CCD_REBIN**2 > 2e-11 / parameters.CCD_REBIN
if parameters.CCD_REBIN == 1:
if parameters.SPECTRACTOR_DECONVOLUTION_PSF2D or parameters.SPECTRACTOR_DECONVOLUTION_FFM:
assert np.isclose(spectrum.lambdas[0], 343, atol=1)
assert np.isclose(spectrum.lambdas[-1], 1084.0, atol=1)
else:
assert np.isclose(spectrum.lambdas[0], 347, atol=1)
assert np.isclose(spectrum.lambdas[-1], 1085.0, atol=1)
assert np.isclose(spectrum.spectrogram_x0, -280, atol=1)
assert np.isclose(spectrum.x0[0] * parameters.CCD_REBIN,
743.6651370068676,
atol=0.5 * parameters.CCD_REBIN)
assert np.isclose(spectrum.x0[1] * parameters.CCD_REBIN,
683.0577836601408,
atol=1 * parameters.CCD_REBIN)
assert 2 < np.mean(
spectrum.chromatic_psf.table['gamma']) * parameters.CCD_REBIN < 3.5
assert os.path.isfile(
os.path.join(output_directory,
tag.replace('.fits', '_spectrum.fits'))) is True
assert os.path.isfile(
os.path.join(output_directory,
tag.replace('.fits', '_spectrogram.fits'))) is True
assert os.path.isfile(
os.path.join(output_directory, tag.replace('.fits',
'_lines.csv'))) is True
def test_logbook():
logbook = LogBook('./ctiofulllogbook_jun2017_v5.csv')
# target, xpos, ypos = logbook.search_for_image('reduc_20170529_085.fits')
# assert xpos is None
disperser_label, target, xpos, ypos = logbook.search_for_image('reduc_20170603_020.fits')
assert target == "PKS1510-089"
assert xpos == 830
assert ypos == 590
def test_astrometry():
file_names = ['tests/data/reduc_20170530_134.fits'
] # 'tests/data/reduc_20170605_028.fits']
load_config('./config/ctio.ini')
logbook = LogBook(logbook='./ctiofulllogbook_jun2017_v5.csv')
parameters.VERBOSE = True
parameters.DEBUG = True
radius = 500
maxiter = 10
for file_name in file_names:
wcs_output_directory = set_wcs_output_directory(file_name)
if os.path.isdir(wcs_output_directory):
subprocess.check_output(f"rm -rf {wcs_output_directory}",
shell=True)
tag = file_name.split('/')[-1].replace('sim', 'reduc')
disperser_label, target, xpos, ypos = logbook.search_for_image(tag)
if target is None or xpos is None or ypos is None:
continue
a = Astrometry(file_name, target, disperser_label)
extent = ((int(max(0, xpos - radius)),
int(min(xpos + radius, parameters.CCD_IMSIZE))),
(int(max(0, ypos - radius)),
int(min(ypos + radius, parameters.CCD_IMSIZE))))
gaia_min_residuals = a.run_full_astrometry(extent=extent,
maxiter=maxiter)
# checks
assert os.path.isdir(wcs_output_directory)
assert os.path.isfile(set_wcs_file_name(file_name))
assert a.data is not None
assert np.sum(a.data) > 1e-10
assert gaia_min_residuals < 0.8
# assert np.all(np.abs([dra_median, ddec_median]) < 1e-3)
if file_name == 'tests/data/reduc_20170605_028.fits':
assert len(a.sources) > 200
assert np.isclose(a.target_radec_position_after_pm.ra.value,
224.97283917)
assert np.isclose(a.target_radec_position_after_pm.dec.value,
-54.30209)
a.my_logger.warning(f"{a.wcs.wcs.crval}")
assert np.isclose(a.wcs.wcs.crval[0], 224.9718998, atol=0.03)
assert np.isclose(a.wcs.wcs.crval[1], -54.28912925, atol=0.03)
if file_name == 'tests/data/sim_20170530_134.fits':
im = Image(file_name, target_label=target)
parameters.SPECTRACTOR_FIT_TARGET_CENTROID = "WCS"
x0_wcs, y0_wcs = find_target(im, guess=[xpos, ypos], rotated=False)
parameters.SPECTRACTOR_FIT_TARGET_CENTROID = "fit"
x0, y0 = find_target(im, guess=[xpos, ypos], rotated=False)
im.my_logger.warning(
f"\n\tTrue {target} position: "
f"{np.array([float(im.header['X0_T']), float(im.header['Y0_T'])])}"
f"\n\tFound {target} position with WCS: {np.array([x0_wcs, y0_wcs])}"
f"\n\tFound {target} position with 2D fit: {np.array([x0, y0])}"
)
assert np.abs(x0_wcs - float(im.header['X0_T'])) < 0.5
assert np.abs(y0_wcs - float(im.header['Y0_T'])) < 1
def test_spectractor():
file_names = ['./tests/data/reduc_20170605_028.fits']
logbook = LogBook(logbook='./ctiofulllogbook_jun2017_v5.csv')
parameters.VERBOSE = True
for file_name in file_names:
tag = file_name.split('/')[-1]
target, xpos, ypos = logbook.search_for_image(tag)
if target is None or xpos is None or ypos is None:
continue
spectrum = Spectractor(file_name, './outputs/', [xpos, ypos], target)
assert spectrum.data is not None
assert os.path.isfile('./outputs/' + tag.replace('.fits', '_spectrum.fits')) is True
def test_fitchromaticpsf2d_run(sim_image="./tests/data/sim_20170530_134.fits",
config="./config/ctio.ini"):
load_test(sim_image, config=config)
tag = sim_image.split('/')[-1]
tag = tag.replace('sim_', 'reduc_')
logbook = LogBook(logbook="./ctiofulllogbook_jun2017_v5.csv")
disperser_label, target, xpos, ypos = logbook.search_for_image(tag)
spectrum = Spectractor(sim_image,
"./tests/data",
guess=[xpos, ypos],
target_label=target,
disperser_label=disperser_label,
config="./config/ctio.ini")
return spectrum
def search_for_image(logbook: LogBook, tag: str) -> (str, int, int):
"""
Search for image according to a `tag`, and return
info (target name, xposition, yposition).
If one of the infos is None, return None.
Parameters
----------
logbook : LogBook
Instance of spectractor.logbook.LogBook
tag : String
The tag of the image (filename) as written in the logbook.
Return
----------
(target, xpos, ypos): (String, Int, Int) or None
Return the info (name, xposition, yposition) or None if
at least one of the infos is None.
"""
target, xpos, ypos = logbook.search_for_image(tag)
if target is None or xpos is None or ypos is None:
return None
else:
return target, xpos, ypos
"--grating",
dest="disperser_label",
default="",
help="Disperser label (default: '').")
args = parser.parse_args()
parameters.VERBOSE = args.verbose
if args.debug:
parameters.DEBUG = True
parameters.VERBOSE = True
file_names = args.input
load_config(args.config)
logbook = LogBook(logbook=args.logbook)
for file_name in file_names:
disperser_label = args.disperser_label
if parameters.OBS_NAME == "CTIO":
tag = file_name.split('/')[-1]
tag = tag.replace('sim_', 'reduc_')
disperser_label, target_label, xpos, ypos = logbook.search_for_image(
tag)
guess = [xpos, ypos]
if target_label is None or xpos is None or ypos is None:
continue
else:
guess = None
if args.target_xy != "0,0":
xpos, ypos = args.target_xy.split(",")
xpos = float(xpos)
parser.add_argument(
"-c",
"--csv",
dest="csv",
default="ctiofulllogbook_jun2017_v5.csv",
help="CSV logbook file. (default: ctiofulllogbook_jun2017_v5.csv).")
args = parser.parse_args()
parameters.VERBOSE = args.verbose
if args.debug:
parameters.DEBUG = True
parameters.VERBOSE = True
file_names = glob.glob(os.path.join(args.input, "*/*.fits"))
logbook = LogBook(logbook=args.csv)
rank = MPI.COMM_WORLD.rank
size = MPI.COMM_WORLD.size
if rank == 0:
print("{} files to process".format(len(file_names)))
print("{} processors used".format(size))
for position in range(rank, len(file_names), size):
file_name = file_names[position]
opt = logbook.search_for_image(file_name.split('/')[-1])
target = opt[0]
xpos = opt[1]
ypos = opt[2]
if target is None or xpos is None or ypos is None:
def fullchain_run(sim_image="./tests/data/sim_20170530_134.fits"):
# load test and make image simulation
load_config("./config/ctio.ini")
if not os.path.isfile(sim_image):
make_image()
image = Image(sim_image)
lambdas_truth = np.fromstring(image.header['LBDAS_T'][1:-1], sep=' ')
amplitude_truth = np.fromstring(image.header['AMPLIS_T'][1:-1],
sep=' ',
dtype=float)
parameters.AMPLITUDE_TRUTH = np.copy(amplitude_truth)
parameters.LAMBDA_TRUTH = np.copy(lambdas_truth)
# extractor
tag = os.path.basename(sim_image)
tag = tag.replace('sim_', 'reduc_')
logbook = LogBook(logbook="./ctiofulllogbook_jun2017_v5.csv")
disperser_label, target, xpos, ypos = logbook.search_for_image(tag)
parameters.PSF_POLY_ORDER = PSF_POLY_ORDER
spectrum = Spectractor(sim_image,
"./tests/data",
guess=[xpos, ypos],
target_label=target,
disperser_label=disperser_label)
# spectrum = Spectrum("./tests/data/sim_20170530_134_spectrum.fits")
# spectrum = Spectrum("./tests/data/sim_20170530_176_spectrum.fits")
# tests
residuals = plot_residuals(spectrum, lambdas_truth, amplitude_truth)
spectrum.my_logger.warning(
f"\n\tQuantities to test:"
f"\n\t\tspectrum.header['X0_T']={spectrum.header['X0_T']:.5g} vs {spectrum.x0[0]:.5g}"
f"\n\t\tspectrum.header['Y0_T']={spectrum.header['Y0_T']:.5g} vs {spectrum.x0[1]:.5g}"
f"\n\t\tspectrum.header['ROT_T']={spectrum.header['ROT_T']:.5g} "
f"vs {spectrum.rotation_angle:.5g}"
f"\n\t\tspectrum.header['BKGD_LEV']={spectrum.header['BKGD_LEV']:.5g} "
f"vs {np.mean(spectrum.spectrogram_bgd):.5g}"
f"\n\t\tspectrum.header['D2CCD_T']={spectrum.header['D2CCD_T']:.5g} "
f"vs {spectrum.disperser.D:.5g}"
f"\n\t\tspectrum.header['A2_FIT']={spectrum.header['A2_FIT']:.5g} vs {A2_T:.5g}"
f"\n\t\tspectrum.header['CHI2_FIT']={spectrum.header['CHI2_FIT']:.4g}"
f"\n\t\tspectrum.chromatic_psf.poly_params="
f"{spectrum.chromatic_psf.poly_params[spectrum.chromatic_psf.Nx + 2 * (PSF_POLY_ORDER + 1):-1]}"
f" vs {PSF_POLY_PARAMS_TRUTH[2 * (PSF_POLY_ORDER + 1):-1]}"
f"\n\t\tresiduals wrt truth: mean={np.mean(residuals[100:-100]):.5g}, "
f"std={np.std(residuals[100:-100]):.5g}")
assert np.isclose(float(spectrum.header['X0_T']), spectrum.x0[0], atol=0.2)
assert np.isclose(float(spectrum.header['Y0_T']), spectrum.x0[1], atol=0.5)
assert np.isclose(float(spectrum.header['ROT_T']),
spectrum.rotation_angle,
atol=180 / np.pi * 1 / parameters.CCD_IMSIZE)
assert np.isclose(float(spectrum.header['BKGD_LEV']),
np.mean(spectrum.spectrogram_bgd),
rtol=1e-2)
assert np.isclose(float(spectrum.header['D2CCD_T']),
spectrum.disperser.D,
atol=0.1)
assert float(spectrum.header['CHI2_FIT']) < 0.65
assert np.all(
np.isclose(
spectrum.chromatic_psf.poly_params[spectrum.chromatic_psf.Nx + 2 *
(PSF_POLY_ORDER + 1):-1],
np.array(PSF_POLY_PARAMS_TRUTH)[2 * (PSF_POLY_ORDER + 1):-1],
rtol=0.1,
atol=0.1))
assert np.abs(np.mean(residuals[100:-100])) < 0.25
assert np.std(residuals[100:-100]) < 2
spectrum_file_name = "./tests/data/sim_20170530_134_spectrum.fits"
# spectrum_file_name = "./tests/data/sim_20170530_176_spectrum.fits"
assert os.path.isfile(spectrum_file_name)
spectrum = Spectrum(spectrum_file_name)
atmgrid_filename = sim_image.replace('sim', 'reduc').replace(
'.fits', '_atmsim.fits')
assert os.path.isfile(atmgrid_filename)
else:
parameters.VERBOSE = False
parameters.DEBUG = False
## Load CTIO image name list
datapath = args.datapath
if args.filesystem == 'hdfs':
file_names = hglob(spark.sparkContext, datapath)
elif args.filesystem == 'lustre':
file_names = glob.glob(os.path.join(datapath, "*/*.fits"))
print("found {} files on {}".format(len(file_names), args.filesystem))
## Name of the logbook containing info about images.
csvfile = args.logfile
logbook = LogBook(logbook=csvfile)
paramdic = {
fn: search_for_image(logbook,
fn.split('/')[-1])
for fn in file_names
}
spark.sparkContext.broadcast(paramdic)
if args.filesystem == 'lustre':
datapath = os.path.join(datapath, "*/*.fits")
rdd = spark.sparkContext.binaryFiles(datapath, len(paramdic))\
.map(lambda x: [x[0].split(":")[1], paramdic[x[0].split(":")[1]], x[1]])\
.filter(lambda x: x[1] is not None)\
.map(lambda x: run_spectractor(x[0], "output", [x[1][1], x[1][2]], x[1][0], x[2]))\