def extract_sources(image,
noise_threshold,
fwhm,
star_finder='DAO',
image_var=None,
background_subtraction=True,
write_to=None,
debug=True):
"""Extract sources from an image with a StarFinder routine.
Long description...
Args:
image (np.ndarray or str):
Image array or the name of a file containing the image array.
noise_threshold (float):
Multiple of the uncertainty/ standard deviation of the image.
fwhm (float):
Expected full width at half maximum (FWHM) of the sources in units of pixels.
star_finder (str, optional):
Choose whether the 'DAO' or 'IRAF' StarFinder implementations from photutils shall be used. Default is
'DAO'.
image_var (float or str):
Variance of the image used for the StarFinder threshold (=noise_threshold * sqrt(image_var)). If not
provided, the code extracts this value from sigma clipped stats. If provided as str-type, the code tries to
use this as a key to the FITS file HDU list.
background_subtraction (bool, optional):
Let the StarFinder consider the background subtraction. Set False for ignoring background flux. Default is
`True`.
write_to (str, optional):
If provided as a str, the list of identified sources is saved to this file.
debug (bool, optional):
Show debugging information. Default is `False`.
Returns:
sources (astropy.table.Table): Table of identified sources, None if no
sources are detected.
"""
# Set logger level
if debug:
logger.setLevel('DEBUG')
# Input parameters
if isinstance(image, np.ndarray):
filename = 'current cube'
elif isinstance(image, str):
logger.info(
"The argument image '{}' is interpreted as file name.".format(
image))
filename = image
image = fits.getdata(filename)
image = image.squeeze()
else:
raise SpecklepyTypeError('extract_sources()',
argname='image',
argtype=type(image),
expected='np.ndarray or str')
# Prepare noise statistics
mean, median, std = sigma_clipped_stats(image, sigma=3.0)
logger.info(
f"Noise statistics for {filename}:\n\tMean = {mean:.3}\n\tMedian = {median:.3}\n\tStdDev = {std:.3}"
)
# Set detection threshold
if image_var is None:
threshold = noise_threshold * std
else:
if isinstance(image_var, str):
# Try to load variance extension from file
image_var = fits.getdata(filename, image_var)
image_var = np.mean(image_var)
threshold = noise_threshold * np.sqrt(image_var)
# Set sky background
if background_subtraction:
logger.info(f"Considering mean sky background of {mean}")
sky = mean
else:
sky = 0.0
# Instantiate StarFinder object
if not isinstance(star_finder, str):
raise SpecklepyTypeError('extract_sources',
argname='starfinder',
argtype=type(star_finder),
expected='str')
if 'dao' in star_finder.lower():
star_finder = DAOStarFinder(fwhm=fwhm, threshold=threshold, sky=sky)
elif 'iraf' in star_finder.lower():
star_finder = IRAFStarFinder(fwhm=fwhm, threshold=threshold, sky=sky)
else:
raise SpecklepyValueError('extract_sources',
argname='star_finder',
argvalue=star_finder,
expected="'DAO' or 'IRAF")
# Find stars
logger.info("Extracting sources...")
sources = star_finder(image)
# Reformatting sources table
sources.sort('flux', reverse=True)
sources.rename_column('xcentroid', 'x')
sources.rename_column('ycentroid', 'y')
sources.keep_columns(['x', 'y', 'flux'])
# Add terminal output
logger.info(f"Extracted {len(sources)} sources")
logger.debug(sources)
# Save sources table to file, if requested
if write_to is not None:
logger.info("Writing list of sources to file {}".format(write_to))
sources.write(write_to, format='ascii.fixed_width', overwrite=True)
return sources
def ssa(files,
mode='same',
reference_file=None,
outfile=None,
in_dir=None,
tmp_dir=None,
lazy_mode=True,
box_indexes=None,
debug=False,
**kwargs):
"""Compute the SSA reconstruction of a list of files.
The simple shift-and-add (SSA) algorithm makes use of the structure of typical speckle patterns, i.e.
short-exposure point-spread functions (PSFs). These show multiple peaks resembling the diffraction-limited PSF of
coherent fractions within the telescope aperture. Under good conditions or on small telescopes, there is typically
one largest coherent atmospheric cell and therefore, speckle PSFs typically show one major intensity peak. The
algorithm makes use of this fact and identifies the emission peak in a given observation frame, assuming that this
always belongs to the same star, and aligns all frames on the coordinate of the emission peak.
See Bates & Cady (1980) for references.
Args:
files (list or array_like):
List of complete paths to the fits files that shall be considered for the SSA reconstruction.
mode (str):
Name of the reconstruction mode: In 'same' mode, the reconstruction covers the same field of view of the
reference file. In 'full' mode, every patch of the sky that is covered by at least one frame will be
contained in the final reconstruction.
reference_file (str, int, optional):
Path to a reference file or index of the file in files, relative to which the shifts are computed. See
specklepy.core.aligment.get_shifts for details. Default is 0.
outfile (specklepy.io.recfile, optional):
Object to write the result to, if provided.
in_dir (str, optional):
Path to the files. `None` is substituted by an empty string.
tmp_dir (str, optional):
Path of a directory in which the temporary results are stored in.
lazy_mode (bool, optional):
Set to False, to enforce the alignment of a single file with respect to the reference file. Default is True.
box_indexes (list, optional):
Constraining the search for the intensity peak to the specified box. Searching the full frames if not
provided.
debug (bool, optional):
Show debugging information. Default is False.
Returns:
reconstruction (np.ndarray):
The image reconstruction. The size depends on the mode argument.
"""
logger.info("Starting SSA reconstruction...")
# Check parameters
if not isinstance(files, (list, np.ndarray)):
if isinstance(files, str):
files = [files]
else:
raise SpecklepyTypeError('ssa()',
argname='files',
argtype=type(files),
expected='list')
if isinstance(mode, str):
if mode not in ['same', 'full', 'valid']:
raise SpecklepyValueError('ssa()',
argname='mode',
argvalue=mode,
expected="'same', 'full' or 'valid'")
else:
raise SpecklepyTypeError('ssa()',
argname='mode',
argtype=type(mode),
expected='str')
if reference_file is None:
reference_file = files[0]
elif isinstance(reference_file, int):
reference_file = files[reference_file]
elif not isinstance(reference_file, str):
raise SpecklepyTypeError('ssa()',
argname='reference_file',
argtype=type(reference_file),
expected='str or int')
if outfile is None:
pass
elif isinstance(outfile, str):
outfile = ReconstructionFile(files=files,
filename=outfile,
cards={"RECONSTRUCTION": "SSA"})
elif isinstance(outfile, ReconstructionFile):
pass
else:
raise SpecklepyTypeError('ssa()',
argname='outfile',
argtype=type(outfile),
expected='str')
if in_dir is None:
in_dir = ''
reference_file = os.path.join(in_dir, reference_file)
if tmp_dir is not None:
if isinstance(tmp_dir, str) and not os.path.isdir(tmp_dir):
os.makedirs(tmp_dir)
if not isinstance(lazy_mode, bool):
raise SpecklepyTypeError('ssa()',
argname='lazy_mode',
argtype=type(lazy_mode),
expected='bool')
if box_indexes is not None:
box = Box(box_indexes)
else:
box = None
if 'variance_extension_name' in kwargs.keys():
var_ext = kwargs['variance_extension_name']
else:
var_ext = 'VAR'
if debug:
logger.setLevel('DEBUG')
logger.handlers[0].setLevel('DEBUG')
logger.info("Set logging level to DEBUG")
# Align reconstructions if multiple files are provided
if lazy_mode and len(files) == 1:
# Do not align just a single file
with fits.open(os.path.join(in_dir, files[0])) as hdu_list:
cube = hdu_list[0].data
if var_ext in hdu_list:
var_cube = hdu_list[var_ext].data
else:
var_cube = None
reconstruction, reconstruction_var = coadd_frames(
cube, var_cube=var_cube, box=box)
else:
# Compute temporary reconstructions of the individual cubes
tmp_files = []
for index, file in enumerate(files):
with fits.open(os.path.join(in_dir, file)) as hdu_list:
cube = hdu_list[0].data
if var_ext in hdu_list:
var_cube = hdu_list[var_ext].data
logger.debug(
f"Found variance extension {var_ext} in file {file}")
else:
logger.debug(
f"Did not find variance extension {var_ext} in file {file}"
)
var_cube = None
tmp, tmp_var = coadd_frames(cube, var_cube=var_cube, box=box)
if debug:
imshow(box(tmp), norm='log')
tmp_file = os.path.basename(file).replace(".fits", "_ssa.fits")
tmp_file = os.path.join(tmp_dir, tmp_file)
logger.info(
"Saving interim SSA reconstruction of cube to {}".format(
tmp_file))
tmp_file_object = Outfile(tmp_file, data=tmp, verbose=True)
# Store variance of temporary reconstruction
if tmp_var is not None:
tmp_file_object.new_extension(var_ext, data=tmp_var)
del tmp_var
tmp_files.append(tmp_file)
# Align tmp reconstructions and add up
file_shifts, image_shape = alignment.get_shifts(
tmp_files,
reference_file=reference_file,
return_image_shape=True,
lazy_mode=True)
pad_vectors, ref_pad_vector = alignment.get_pad_vectors(
file_shifts,
cube_mode=(len(image_shape) == 3),
return_reference_image_pad_vector=True)
# Iterate over file-wise reconstructions
reconstruction = None
reconstruction_var = None
for index, file in enumerate(tmp_files):
# Read data
with fits.open(file) as hdu_list:
tmp_image = hdu_list[0].data
if var_ext in hdu_list:
tmp_image_var = hdu_list[var_ext].data
else:
tmp_image_var = None
# Initialize or co-add reconstructions and var images
if reconstruction is None:
reconstruction = alignment.pad_array(
tmp_image,
pad_vectors[index],
mode=mode,
reference_image_pad_vector=ref_pad_vector)
if tmp_image_var is not None:
reconstruction_var = alignment.pad_array(
tmp_image_var,
pad_vectors[index],
mode=mode,
reference_image_pad_vector=ref_pad_vector)
else:
reconstruction += alignment.pad_array(
tmp_image,
pad_vectors[index],
mode=mode,
reference_image_pad_vector=ref_pad_vector)
if tmp_image_var is not None:
reconstruction_var += alignment.pad_array(
tmp_image_var,
pad_vectors[index],
mode=mode,
reference_image_pad_vector=ref_pad_vector)
logger.info("Reconstruction finished...")
# Save the result to an Outfile
if outfile is not None:
outfile.data = reconstruction
if reconstruction_var is not None:
outfile.new_extension(name=var_ext, data=reconstruction_var)
# Return reconstruction (and the variance map if computed)
if reconstruction_var is not None:
return reconstruction, reconstruction_var
return reconstruction
def main():
# Parse args
parser = GeneralArgParser()
args = parser.parse_args()
if args.debug:
logger.setLevel('DEBUG')
logger.debug(args)
if args.gui:
start()
# Execute the script of the corresponding command
if args.command is 'generate':
# Read parameters from file and generate exposures
target, telescope, detector, parameters = get_objects(args.parfile,
debug=args.debug)
generate_exposure(target=target,
telescope=telescope,
detector=detector,
debug=args.debug,
**parameters)
elif args.command is 'reduce':
# In setup mode
if args.setup:
run.setup(path=args.path,
instrument=args.instrument,
par_file=args.parfile,
list_file=args.filelist,
sort_by=args.sortby)
# Else start reduction following the parameter file
else:
params = config.read(args.parfile)
run.full_reduction(params, debug=args.debug)
elif args.command is 'ssa':
# Prepare path information and execute reconstruction
if args.tmpdir is not None and not os.path.isdir(args.tmpdir):
os.mkdir(args.tmpdir)
ssa(args.files,
mode=args.mode,
tmp_dir=args.tmpdir,
outfile=args.outfile,
box_indexes=args.box_indexes,
debug=args.debug)
elif args.command is 'holography':
# Read parameters from file and execute reconstruction
defaults_file = os.path.join(os.path.dirname(__file__),
'../config/holography.cfg')
defaults_file = os.path.abspath(defaults_file)
params = config.read(defaults_file)
params = config.update_from_file(params, args.parfile)
holography(params,
mode=params['OPTIONS']['reconstructionMode'],
debug=args.debug)
elif args.command is 'aperture':
if args.mode == 'psf1d':
logger.info("Extract 1D PSF profile")
analysis.get_psf_1d(args.file,
args.index,
args.radius,
args.out_file,
args.normalize,
debug=args.debug)
elif args.mode == 'variance':
logger.info("Extract 1D PSF variation")
analysis.get_psf_variation(args.file, args.index, args.radius,
args.out_file, args.normalize,
args.debug)
else:
logger.warning(f"Aperture mode {args.mode} not recognized!")
elif args.command is 'extract':
if args.out_file is None:
args.out_file = 'sources_' + os.path.basename(
args.file_name).replace('.fits', '.dat')
extract_sources(image=args.file_name,
noise_threshold=args.noise_threshold,
fwhm=args.fwhm,
image_var=args.var,
write_to=args.out_file)
elif args.command == 'plot':
plot = Plot.from_file(file_name=args.file,
extension=args.extension,
columns=args.columns,
format=args.format,
layout=args.layout,
debug=args.debug)
plot.apply_layout(layout=args.layout)
plot.save()
plot.show()
elif args.command is 'apodization':
get_resolution_parameters(wavelength=args.wavelength,
diameter=args.diameter,
pixel_scale=args.pixel_scale)
def get_objects(parameter_file, debug=False):
"""Get objects from parameter file.
Args:
parameter_file (str):
File from which the objects are instantiated.
debug (bool, optional):
Show debugging information.
Returns:
target (Target object):
telescope (Telescope object):
detector (Detector object):
parameters (dict):
Dictionary containing the parameters parsed toward generate exposure beyond the three objects.
"""
# Set logger level
if debug:
logger.setLevel('DEBUG')
# Check whether files exist
if not os.path.isfile(parameter_file):
raise FileNotFoundError(f"Parameter file {parameter_file} not found!")
# Read parameter file
params = config.read(parameter_file)
# Create objects from the parameters
target = Target(**params['TARGET'])
logger.debug(f"Initialized Target instance:\n{target}")
telescope = Telescope(**params['TELESCOPE'])
logger.debug(f"Initialized Telescope instance:\n{telescope}")
detector = Detector(**params['DETECTOR'])
logger.debug(f"Initialized Detector instance:\n{detector}")
# Extract and interpret other kez word parameters
if 'KWARGS' in params:
parameters = params['KWARGS']
elif 'PARAMETERS' in params:
parameters = params['PARAMETERS']
else:
parameters = {}
# Interpret str-type key word arguments
for key in parameters.keys():
if isinstance(parameters[key], str):
try:
parameters[key] = eval(parameters[key])
logger.debug(
f"Kwarg {key} evaluated as {parameters[key]} ({type(parameters[key])})"
)
except SyntaxError:
parameters[key] = Quantity(parameters[key])
logger.debug(
f"Kwarg {key} evaluated as {parameters[key]} ({type(parameters[key])})"
)
except NameError:
logger.debug(
f"Kwarg {key} not evaluated {parameters[key]} ({type(parameters[key])})"
)
return target, telescope, detector, parameters
def full_reduction(params, debug=False):
"""Execute a full reduction following the parameters in the `params` dictionary.
TODO: Split this function into the parts and sort into the other modules
Args:
params (dict):
Dictionary with all the settings for reduction.
debug (bool, optional):
Show debugging information.
"""
# Set logging level
if debug:
logger.setLevel('DEBUG')
# (0) Read file list table
logger.info("Reading file list ...")
in_files = FileArchive(file_list=params['PATHS']['fileList'],
in_dir=params['PATHS']['filePath'],
out_dir=params['PATHS']['outDir'])
logger.info('\n' + str(in_files.table))
# (1) Initialize directories and reduction files
if not os.path.isdir(params['PATHS']['outDir']):
os.makedirs(params['PATHS']['outDir'])
if not os.path.isdir(params['PATHS']['tmpDir']):
os.makedirs(params['PATHS']['tmpDir'])
if 'skip' in params['PATHS'] and params['PATHS']['skip']:
product_files = glob.glob(os.path.join(params['PATHS']['outDir'], '*fits'))
else:
product_files = in_files.initialize_product_files(prefix=params['PATHS']['prefix'])
# (2) Flat fielding
if 'skip' in params['FLAT'] and params['FLAT']['skip']:
logger.info('Skipping flat fielding as requested from parameter file...')
else:
flat_files = in_files.get_flats()
if len(flat_files) == 0:
logger.warning("Did not find any flat field observations. No flat field correction will be applied!")
else:
logger.info("Starting flat field correction...")
master_flat = flat.MasterFlat(flat_files, file_name=params['FLAT']['masterFlatFile'],
file_path=params['PATHS']['filePath'], out_dir=params['PATHS']['tmpDir'])
master_flat.combine()
master_flat.run_correction(file_list=product_files, file_path=None)
# (3) Linearization
# TODO: Implement linearization
# (4) Sky subtraction
if 'skip' in params['SKY'] and params['SKY']['skip']:
logger.info('Skipping sky background subtraction as requested from parameter file...')
else:
logger.info("Starting sky subtraction...")
try:
sky.subtract_sky_background(**params['SKY'], in_files=in_files, out_files=product_files,
file_path=params['PATHS']['filePath'], tmp_dir=params['PATHS']['tmpDir'])
except RuntimeError as e:
raise RuntimeWarning(e)
# Close reduction
logger.info("Reduction finished...")
def subtract_sky_background(in_files,
out_files=None,
method='scalar',
source='sky',
mask_sources=False,
file_path=None,
tmp_dir=None,
show=False,
debug=False):
"""Estimate and subtract the sky background via different methods and sources.
TODO: Implement sky subtraction from image
Args:
in_files (specklepy.FileArchive):
File archive storing the information of all the files in the reduction.
out_files (list):
List of files to apply the sky subtraction to. If left empty, the list stored in the `in_files` FileArchive
is used.
method (str, optional):
Switch between a scalar (`scalar`) background value or a 2D image (`images`).
source (str, optional):
Source for estimating the background from. Can be either `sky` to measure from dedicated sky frames or
`science` to use the science frames themselves. Typically, these frames have a high number of sources, so
`mask_sources` should be switched on.
mask_sources (bool, optional):
In empty reference fields, this masking option should stay at `False`, since source masking is not well
tested. However, masking sources yields a more precise result.
file_path (str, optional):
Path to the files, listed in `in_files`.
tmp_dir (str, optional):
Directory to which temporary results and QA data is stored.
show (bool, optional):
Show plots of sky estimates for each sequence. They will be created and stored regardless of this choice.
debug (bool, optional):
Show debugging information.
"""
# Set logging level
if debug:
logger.setLevel('DEBUG')
# Apply fall back values
if method is None:
method = 'scalar'
logger.info(f"Sky background subtraction method: {method}")
if source is None:
source = 'sky'
logger.info(f"Sky background subtraction source: {source}")
if out_files is None:
out_files = in_files.product_files
if out_files is None:
logger.warning(
f"Output files are not declared in subtract_sky_background!")
# Identify the observing sequences
sequences = in_files.identify_sequences(source=source)
# Start the background estimates
if method == 'scalar':
# Iterate through observing sequences
for s, sequence in enumerate(sequences):
logger.info(
f"Starting observing sequence {s} :: Object {sequence.object} :: Setup {sequence.setup}"
)
# Compute weights based on the time offset to the individual sky observations
weights = sequence.compute_weights()
# Start extracting sky fluxes
sky_bkg = np.zeros(sequence.n_sky)
sky_bkg_std = np.zeros(sequence.n_sky)
for i in trange(sequence.n_sky,
desc='Estimate sky background from cube'):
file = sequence.sky_files[i]
bkg, d_bkg = estimate_sky_background(file,
method=method,
mask_sources=mask_sources,
path=file_path)
sky_bkg[i] = bkg
sky_bkg_std[i] = d_bkg
logger.debug(
f"Shapes:\nF: {sky_bkg.shape}\ndF: {sky_bkg_std.shape}")
# Compute weighted sky background for each science file
weighted_sky_bkg = np.dot(weights, sky_bkg)
weighted_sky_bkg_var = np.dot(np.square(weights),
np.square(sky_bkg_std))
# Store sky background estimates
sky_bkg_table = Table(data=[
sequence.sky_files, weighted_sky_bkg, weighted_sky_bkg_var
],
names=['FILE', 'BKG', 'VAR'])
sky_bkg_table_name = f"sky_bkg_{sequence.object}_{sequence.setup}.fits"
sky_bkg_table.write(os.path.join(tmp_dir, sky_bkg_table_name),
overwrite=True)
# Plot sky flux estimates
for i, file in enumerate(sequence.sky_files):
plt.text(sequence.sky_time_stamps[i],
sky_bkg[i],
file,
rotation=90,
alpha=.5)
for i, file in enumerate(sequence.science_files):
plt.text(sequence.science_time_stamps[i],
weighted_sky_bkg[i],
file,
rotation=90,
alpha=.66)
plt.errorbar(x=sequence.sky_time_stamps,
y=sky_bkg,
yerr=sky_bkg_std,
fmt='None',
ecolor='tab:blue',
alpha=.5)
plt.plot(sequence.sky_time_stamps,
sky_bkg,
'D',
label='Sky',
c='tab:blue')
plt.errorbar(x=sequence.science_time_stamps,
y=weighted_sky_bkg,
yerr=np.sqrt(weighted_sky_bkg_var),
fmt='None',
ecolor='tab:orange',
alpha=.66)
plt.plot(sequence.science_time_stamps,
weighted_sky_bkg,
'D',
label='Science',
c='tab:orange')
plt.xlabel('Time (s)')
plt.ylabel('Flux (counts)')
plt.legend()
save_figure(
os.path.join(tmp_dir,
sky_bkg_table_name.replace('.fits', '.png')))
if show:
plt.show()
plt.close()
# Subtract sky from product files
for i, science_file in enumerate(sequence.science_files):
for out_file in out_files:
if science_file in out_file:
science_file = out_file
logger.info(
f"Applying sky background subtraction on file {science_file}"
)
with fits.open(science_file, mode='update') as hdu_list:
hdu_list[0].data = hdu_list[0].data.astype(
float) - weighted_sky_bkg[i]
if 'VAR' in hdu_list:
hdu_list['VAR'].data = hdu_list[
'VAR'].data + weighted_sky_bkg_var[i]
else:
# Construct new HDU
shape = np.array(hdu_list[0].data.shape)[[-2, -1]]
data = np.full(shape=shape,
fill_value=weighted_sky_bkg_var[i])
hdu = fits.ImageHDU(data=data, name='VAR')
hdu_list.append(hdu)
hdu_list[0].header.set('SKYCORR', str(datetime.now()))
hdu_list[0].header.set('SKYBKG', weighted_sky_bkg[i],
"Sky background")
hdu_list[0].header.set('SKYVAR', weighted_sky_bkg_var[i],
"Sky background variance")
hdu_list.flush()
elif method in ['image', 'frame']:
raise NotImplementedError(
"Sky subtraction in image mode is not implemented yet!")
else:
raise ValueError(f"Sky subtraction method {method} is not understood!")