def combineFlats(flatlist, dark=None, bias=None):
"""Combine all flat files into a flat master. Subtract dark or bias if provided."""
ccdflatlist = [
ccdproc.CCDData.read(aflat, unit="adu") for aflat in flatlist
]
if dark is not None and bias is None:
flat_sub = [
ccdproc.subtract_dark(aflat,
dark,
exposure_time='exptime',
exposure_unit=u.second)
for aflat in ccdflatlist
]
elif dark is None and bias is not None:
flat_sub = [
ccdproc.subtract_bias(aflat, bias) for aflat in ccdflatlist
]
else:
flat_sub = ccdflatlist
flatComb = ccdproc.Combiner(flat_sub)
flatComb.sigma_clipping(low_thresh=3, high_thresh=3, func=np.ma.median)
flatComb.scaling = lambda arr: 1. / np.ma.average(arr)
flatmaster = flatComb.average_combine()
return flatmaster
def create_masterbias(image_list):
"""Create a master bias frame from a list of images
Parameters
----------
image_list: list
List contain the file names to be processed
Returns
-------
masterbias: ccddata.CCDData
Combine master bias from the biases supplied in image_list
"""
# determine whether they are red or blue
if os.path.basename(image_list[0]).startswith('H'):
func = blue_process
elif os.path.basename(image_list[0]).startswith('R'):
func = red_process
else:
raise TypeError('These are not standard HRS frames')
# reduce the files
ccd_list = []
for image_name in image_list:
ccd = func(image_name, masterbias=None, error=False)
ccd_list.append(ccd)
# combine the files
cb = ccdproc.Combiner(ccd_list)
nccd = cb.median_combine(median_func=np.median)
return nccd
def combineBias(biaslist):
"""Combine all the bias files into a master bias"""
ccdlist = [ccdproc.CCDData.read(abias, unit="adu") for abias in biaslist]
biasComb = ccdproc.Combiner(ccdlist)
biasComb.sigma_clipping(low_thresh=3, high_thresh=3, func=np.ma.median)
biasmaster = biasComb.average_combine()
return biasmaster
def combineDarks(darklist):
"""Combine all the dark files into a master dark"""
darkComb = ccdproc.Combiner(
[ccdproc.CCDData.read(adark, unit="adu") for adark in darklist])
darkComb.sigma_clipping(low_thresh=3, high_thresh=3, func=np.ma.median)
darkmaster = darkComb.average_combine()
darkmaster.header['exptime'] = fits.getval(darklist[0], 'exptime')
return darkmaster
def overscan_trim_and_sigma_clip_median(image_list, clip_baseline_func=med_over_images):
"""
Combine a list of images using median
This function does several steps:
1. Subtract overscan
2. Trim image
3. sigma clip image using a median of the unclipped stack as the baseline
4. combine the images on the list using median
** It modifies the images in the input list. **
"""
oscan_and_trim(image_list)
combo = ccdproc.Combiner(image_list)
combo.sigma_clipping(func=clip_baseline_func)
return combo
def combine_files(filelist, outputfile, directory=None):
# take a list of input files
if os.path.exists(filelist):
# load the files
infiles = np.loadtxt(filelist, dtype=str)
inputccd = []
for i in xrange(len(infiles)):
if directory is not None:
filename = os.path.join(directory, infiles[i])
else:
filename = infiles[i]
hdu = fits.open(filename, ignore_missing_end=True)
ccd = CCDData(np.copy(hdu[0].data), unit=u.adu)
inputccd.append(ccd)
combo = ccdproc.Combiner(inputccd)
output = combo.average_combine()
fits.writeto(outputfile, output.data)
def _action_for_one_group(self, filter_dict=None):
combined_dict = self.apply_to.copy()
if filter_dict is not None:
combined_dict.update(filter_dict)
images = []
for hdu in self.image_source.hdus(**combined_dict):
try:
unit = hdu.header['BUNIT']
except KeyError:
unit = DEFAULT_IMAGE_UNIT
images.append(
ccdproc.CCDData(data=hdu.data, meta=hdu.header, unit=unit))
combiner = ccdproc.Combiner(images, dtype=images[0].dtype)
if self._clipping_widget.toggle.value:
if self._clipping_widget.min_max:
combiner.minmax_clipping(
min_clip=self._clipping_widget.min_max.min,
max_clip=self._clipping_widget.min_max.max)
if self._clipping_widget.sigma_clip:
combiner.sigma_clipping(
low_thresh=self._clipping_widget.sigma_clip.min,
high_thresh=self._clipping_widget.sigma_clip.max,
func=np.ma.median)
if self._combine_method.scaling_func:
combiner.scaling = self._combine_method.scaling_func
if self._combine_method.method == 'Average':
combined = combiner.average_combine()
elif self._combine_method.method == 'Median':
combined = combiner.median_combine()
combined.header = images[0].header
combined.header['master'] = True
if combined.data.dtype != images[0].dtype:
combined.data = np.array(combined.data, dtype=images[0].dtype)
return combined
def clean_the_images(path, filename):
#ast=AstrometryNet()
#ast.api_key= 'iqmqwvazpvolmjmn'
dir = path
gain = 2 * u.electron / u.adu
readnoise = 7.5 * u.electron
ra = input('Enter the RA of the source: ')
dec = input('Enter the DEC of the source: ')
'''
wcs_header=ast.solve_from_image(path+filename)
wcs=WCS(wcs_header)
ran,decn=wcs.all_pix2world(1024,1024,0)
print(ran,decn)
'''
file_name = os.path.join(dir, filename)
image = ccdproc.CCDData.read(file_name, unit='adu')
header = fits.getheader(file_name, 0)
time = header['DATE']
t = Time(time, format='isot', scale='utc')
print(t.jd, t.mjd)
header.insert(15, ('RA', ra))
header.insert(16, ('DEC', dec))
a = sorted(glob(os.path.join(dir, 'bias*.fits')))
biaslist = []
for i in range(0, len(a)):
data = ccdproc.CCDData.read(a[i], unit='adu')
#data = ccdproc.create_deviation(data, gain=gain, readnoise=readnoise)
#data= data-(data.uncertainty.array)
biaslist.append(data)
combiner = ccdproc.Combiner(biaslist)
masterbias = combiner.median_combine()
masterbias.write('masterbias.fit', overwrite=True)
mbias = ccdproc.CCDData.read('masterbias.fit', unit='adu')
#masterbias.meta=image.meta
print('master bias generated')
print(np.mean(masterbias), np.median(masterbias))
c = sorted(glob(os.path.join(dir, 'flat*.fits')))
flatlist = []
for j in range(0, len(c)):
flat = ccdproc.CCDData.read(c[j], unit='adu')
#flat= ccdproc.create_deviation(flat, gain=gain, readnoise=readnoise)
flat = ccdproc.subtract_bias(flat, masterbias)
flatlist.append(flat)
combiner = ccdproc.Combiner(flatlist)
masterflat = combiner.median_combine()
masterflat.write('masterflat.fits', overwrite=True)
mflat = ccdproc.CCDData.read('masterflat.fits', unit='adu')
print('master flat generated')
print(np.mean(masterflat), np.median(masterflat))
#masterflat.meta=image.meta
bias_subtracted = ccdproc.subtract_bias(image, masterbias)
flat_corrected = ccdproc.flat_correct(bias_subtracted, masterflat)
cr_cleaned = ccdproc.cosmicray_lacosmic(flat_corrected,
readnoise=7.5,
sigclip=5)
print('cosmic ray removed')
fits.writeto(dir + 'j_0947_i_1_clean.fits',
cr_cleaned,
header,
overwrite=True)
print('image cleaned')
def combine_list_to_file(listname,
outname,
read_from_file=False,
combine='median',
datahdus=0):
if read_from_file == True:
listfile = open(listname, 'r')
flist = listfile.read().splitlines()
listfile.close()
else:
flist = listname
if len(flist) == 0:
mylog('Failed write, trying to overwrite: {0}'.format(listname))
return
# combine_list will be a list of lists of data objects
combine_list = []
ifirst = True
for line in flist:
fname = line.strip()
hdulist = fits.open(fname)
if ifirst == True:
try:
# This should propagate header and any HDUs that don't
# get combined (ie are not in datahdus list) to the output.
hdulist.writeto(outname)
except:
mylog('Failed write, trying to overwrite: {0}'.format(outname))
hdulist.writeto(outname, overwrite=True)
ifirst = False
nhdus = len(hdulist)
# Data of this image will be listed in a single entry in combine_list
tmplist = []
if nhdus > 1:
istart = 1
else:
istart = 0
hduindexes = list(range(nhdus))[istart:]
if datahdus != 0:
hduindexes = datahdus
for i in hduindexes:
data1 = ccdproc.CCDData(hdulist[i].data, unit="adu")
head1 = hdulist[i].header
head1['filename'] = fname
data1.header = head1
tmplist.append(data1)
# combine_list.append(ccdproc.CCDData(data=data1, meta=head1, unit="adu"))
hdulist.close()
combine_list.append(tmplist)
for i in hduindexes:
# Take the i-1'th data extension (0-based) from each image and
# put these into a list to combine
tmplist = [elem[i - 1] for elem in combine_list]
combo = ccdproc.Combiner(tmplist)
if combine == 'average':
output1 = combo.average_combine()
else:
output1 = combo.median_combine()
# Header output by combine is minimal and causes failure
# fits.update(outname, output1.data, header=output1.header, ext=i)
# but outputting without header field leaves only 6 line header
# fits.update(outname, output1.data, ext=i)
# copying header from the first input image works.
fits.update(outname, output1.data, header=tmplist[0].header, ext=i)
mylog("Combined to output file {0}".format(outname))
return
def reduce_image(
bads: Iterable[str],
flats: Iterable[str],
images: Iterable[str],
band_id: Band = Band.J, # TODO: allow 'all'
combine: str = 'median',
skyscale_method: str = 'subtract',
single_thread: bool = False) -> CCDData:
"""
Take a list of files for badPixel, flatfield and exposures + a bunch of processing parameters and reduce them
to write an output filec
:param bads: list of paths to bad pixel frames
:param flats: list of paths to flat frames
:param images: list of paths to images
:param band_id: which spectral band to look at, enum
:param combine: either 'median' or 'average'
:param skyscale_method: either 'subtract' or 'divide'
:param single_thread: if false, don't use multiprocessing pool
:return: (combined_output, scamp_output, sextractor_output)
"""
assert band_id
# you can't just reasign the outer scope variable without weirdness...
_Pool = PoolDummy if single_thread else Pool
# use pool as a context manager so that terminate() gets called automatically
with _Pool(n_cpu) as pool:
read_files = read_and_sort(bads, flats, images, pool)[band_id]
if not (len(read_files.flat) > 0 and len(read_files.images) > 0):
raise ValueError(
'cannot continue, not enough data left after filtering data by available spectral band'
)
dimensions = read_files.flat[0].shape
for image in itertools.chain(read_files.bad, read_files.flat,
read_files.images):
if not image.shape == dimensions:
raise ValueError('image dimension mismatch', image)
# TODO distortion correct here
# Perform basic reduction operations
processed = standard_process(read_files.bad, read_files.flat[0],
read_files.images, pool)
skyscaled = skyscale(processed, skyscale_method, pool)
# does only make sense when we have bad pixels
fixed = pool.map(fix_pix, skyscaled) if bads else skyscaled
# Reproject everything to the world-coordinate system of the first image
wcs = fixed[0].wcs
reprojected = pool.starmap(ccdproc.wcs_project,
zip(fixed, itertools.repeat(wcs)))
# TODO option to align with cross correlation (see image_registration)
# overlay images
combiner = ccdproc.Combiner(reprojected)
output_image = combiner.median_combine(
) if combine == 'median' else combiner.average_combine()
# WTF. ccdproc.Combiner is not giving back good metadata.
# need to set WCS manually again and convert header to astropy.fits.io.Header object from an ordered dict
# not replacing this can cause weird errors during file writing/wcs conversion
output_image.wcs = wcs
output_image.header = astropy.io.fits.header.Header(
output_image.header)
return output_image
def stacking_procedure(images: [Image]):
"""stacks images in list into single one and writes file
header will be from middle image in the list.
Time will be between start of first and end of last exposure with exposure length of this difference.
:param images list of Image list objects to stack
"""
# go through images and collect info
names = []
i = 0
half = len(images) // 2
tfirst = images[0].get_time_jd()
tlast = images[-1].get_time_jd() + images[-1].get_exposure() / 86000
timejd_mid = (tfirst + tlast) / 2
timecoverage = (tlast - tfirst) * 86000 # s
for image in images: # get stack names
if "stack" in image.processing_parameters:
for name in image.get_stack():
names.append(name)
else:
names.append(str(image.get_path()))
i += 1
if i == half:
header = fits.getheader(image.get_path())
midpoint_WCS = WCS(header)
reprojected = []
# reproject all images onto the middle one
for image in images:
data = fits.getdata(image.get_path())
header_wcs = fits.getheader(image.get_path())
ccddata = ccdproc.CCDData(data, wcs=WCS(header_wcs), unit="adu")
reprojected.append(wcs_project(ccddata, midpoint_WCS))
combiner = ccdproc.Combiner(reprojected)
final_image_data = combiner.average_combine()
final_image_data.wcs = WCS(header)
header["EXPTIME"] = timecoverage
header["EXPOSURE"] = timecoverage
header["JD"] = tfirst
filename = "stack-" + str(timejd_mid) + "-e" + "{:.0f}".format(
timecoverage) + ".fits"
path = Path("/tmp")
path = path / filename
if os.path.exists(path):
os.remove(path)
fits.writeto(path, final_image_data, header)
stacked_image = Image(fixed_parameters={
"path": path,
"exposure": timecoverage,
"time_jd": tfirst,
"type": "data",
"id": filename
},
processing_parameters={
"flat": True,
"dark": True,
"stack": names
})
return stacked_image
def make_mflat(file_list,
mbias_path,
out_path,
filter,
scaling_func=_center_inv_median):
"""
Given a list of flat image files, combine then into master flat using
sigma clipping algorithm, on the images after normalizing by the median. It
is expected that the files are already pre-processed (overscan, trim, ...).
Arguments
---------
file_list : list of str
Paths for the files.
mbias_path : str
Path to the master bias
out_path : pathlib.Path
Location to put new image
filter : str
Name of filter (used to generate out filename)
scaling_func : function or list of values
Function or values to scale individual images. Default is
_center_inv_median, which is defined on this module.
File transformations
--------------------
Re-write FITS files, with overscan correction and updated header.
Returns
-------
out_pathname : str
Path to the generated file.
"""
out_pathname = out_path / f"master_flat_{filter}.fits"
if out_pathname.exists():
print(
f"File: {str(out_pathname)} already exists. Exiting the function."
)
return {filter: out_pathname}
else:
out_pathname = str(out_pathname)
# Checking first HDUList for the image extensions
with fits.open(file_list[0]) as hdus:
image_indices = _check_image_extensions(hdus)
# Generating dummy file using the first image
mflat = fits.open(file_list[0])
mflat.writeto(out_pathname)
# Looping over extensions
for i in image_indices:
master_bias = CCDData.read(mbias_path, hdu=i,
unit="adu") # Master bias
# Loading images of an extension
ccd_list = [
CCDData.read(image_file, hdu=i, unit="adu")
for image_file in file_list
]
for image in ccd_list:
image = ccdproc.subtract_bias(image, master_bias)
if not ('BIASSEC' in ccd_list[0].header):
print(
f".Skipping flat overscan correction on: index [{i:1.0f}] - BIASSEC keyword not found"
)
else:
for im in ccd_list:
im = _correct_overscan_hdu(im)
# Combining images
comb = ccdproc.Combiner(ccd_list)
comb.scaling = scaling_func # Scalling using custom function
comb.sigma_clipping(low_thresh=3, high_thresh=3, func=np.ma.median)
comb_image = comb.average_combine()
# Writing changes to the master bias file
mflat[i].header.append(
('NCOMBINE', len(ccd_list), '# images combined'))
fits.update(out_pathname,
comb_image.data,
header=mflat[i].header,
ext=i)
mflat.close()
print(f"Processed master flat on {filter} : {out_pathname}")
return {filter: out_pathname}
def make_mbias(file_list, out_path):
"""
Given a list of bias image files, combine then into master bias using
sigma clipping algorithm. It is expected that the files are already
pre-processed (overscan, trim, ...).
Arguments
---------
file_list : list of str
Paths for the files.
out_path : pathlib.Path
Location to put new image
File transformations
--------------------
Write master bias FITS on out_pathname
Returns
-------
out_pathname : str
Path to the generated file.
"""
out_pathname = out_path / "master_bias.fits"
if out_pathname.exists():
print(
f"File: {str(out_pathname)} already exists. Exiting the function."
)
return str(out_pathname)
else:
out_pathname = str(out_pathname)
# Checking HDU for the image extensions
with fits.open(file_list[0]) as hdus:
image_indices = _check_image_extensions(hdus)
# Generating dummy file using the first bias image
mbias = fits.open(file_list[0])
mbias.writeto(out_pathname)
# Looping over extensions
for i in image_indices:
# Loading images of an extension
ccd_list = [
CCDData.read(image_file, hdu=i, unit="adu")
for image_file in file_list
]
# Correct overscan
if not ('BIASSEC' in ccd_list[0].header):
print(
f".Skipping bias overscan correction on: index [{i:1.0f}] - BIASSEC keyword not found"
)
else:
for im in ccd_list:
im = _correct_overscan_hdu(im)
# Combining images
comb = ccdproc.Combiner(ccd_list)
comb.sigma_clipping(low_thresh=3, high_thresh=3, func=np.ma.median)
comb_bias = comb.average_combine()
# Writing changes to the master bias file
mbias[i].header.append(
('NCOMBINE', len(ccd_list), '# images combined'))
fits.update(out_pathname,
comb_bias.data,
header=mbias[i].header,
ext=i)
mbias.close()
print(f"Processed master bias: {out_pathname}")
return out_pathname
def reduce_night(science_collection, dark_collection, flat_collection, config, config_arguments):
"""
This function reduce science data of one night and save the results to a folder named "reduced". The reduction
is performed as follows:
- Create a list of masterdarks (each masterdark has a different value of the exposure time) ^1
- Create a list of masterflats (each masterflat has a different value of the filter) ^1
- Reduce the science data as follows:
*For each filter:
*For each exposure time with that filter:
-- Look for the masterdark with the nearest exposure time
-- Look for the masterflat with the current filter.
-- Substract masterdark
-- Flat field correct the data
-- Clean cosmic rays (if requested)
-- Save image to ./Calibrated folder
(1) The master(flat/dark)s are created using mean combine.
:param science_collection: Numpy array - A numpy array with the science collection data produced by FitsLookup.
:param dark_collection: Numpy array - A numpy array with the dark collection data produced by FitsLookup.
:param flat_collection: Numpy array - A numpy array with the flat collection data produced by FitsLookup.
:param config_values: Dictionary - Dictionary - A dictionary provided by the function get_config_dict that
contains the config of the fits files ( readed from conf.INI ).
:param config_arguments: Dictionary - A dictionary provided by argparse initialization that contain the current flags.
:return: Integer - 0 if no errors raised 1 if errors raised.
"""
# Supress astropy warnings
warnings.filterwarnings('ignore')
# Renaming some config_arguments for easy acess
work_dir = config_arguments.dir[0]
# Get the filter and exposure collection of science and flat images
science_filter_collection = set(science_collection['filter'])
science_exposures_collection = set(science_collection['exptime'])
dark_exposures_collection = set(dark_collection['exptime'])
flat_filter_collection = set(flat_collection['filter'])
# Inform the user of the filter / exptime found.
science_exp_times_as_string = ", ".join(
[str(x) for x in science_exposures_collection])
dark_exp_times_as_string = ", ".join(
[str(x) for x in dark_exposures_collection])
module_logger.info("We have found {0} filters in the science images: {1}".format(
len(science_filter_collection), ", ".join(science_filter_collection)))
module_logger.info("We have found {0} exposure times science images: {1}".format(
len(science_exposures_collection), science_exp_times_as_string))
module_logger.info("We have found {0} exposure times dark calibrators: {1}".format(
len(dark_exposures_collection), dark_exp_times_as_string))
module_logger.info("We have found {0} filters in the flat calibrators {1}".format(
len(flat_filter_collection), ", ".join(flat_filter_collection)))
# Check if we have the same filters in flats and science, if not, get the
# intersection
if not science_filter_collection.issubset(flat_filter_collection):
module_logger.warning(
"There are more filters in the science images than in the flat calibrators")
science_filter_collection = science_filter_collection.intersection(
flat_filter_collection)
module_logger.warning("Triying to work with common filters.")
module_logger.info("We have found {0} common filters in the science images: {1}".format(
len(science_filter_collection), ", ".join(science_filter_collection)))
if not science_filter_collection:
module_logger.warning(
"There are no common filters between science images and flat calibrators")
module_logger.warning("This night will be skiped.")
return 1
# Warn the user if we found science images of 0 seconds
if 0 in science_exposures_collection:
number_of_null_images = len(filter_collection(
science_collection, [('exptime', 0)]))
module_logger.warning(
"We have found {0} science images with 0 seconds of exposure time.".format(number_of_null_images))
science_exposures_collection.discard(0)
module_logger.warning(
"Discarding images with 0 seconds of exposure time for this night: {0} exposure(s) remain.".format(
len(science_exposures_collection)))
# ------- MASTER DARK CREATION --------
module_logger.info("Starting the creation of the master dark")
module_logger.info("{0} different exposures for masterdarks were found.".format(
len(dark_exposures_collection)))
master_dark_collection = dict()
# Loop over each exposure time.
for dark_exposure_item in dark_exposures_collection:
module_logger.info("Creating masterdark with exposure of {0}s".format(dark_exposure_item))
# Initializate dark list for current collection.
exposure_dark_list = list()
for dark_image_data in filter_collection(dark_collection, [('exptime', dark_exposure_item)]):
# Open the images and append to the dark list
dark_image = dark_image_data['filename']
ccd = CCDData.read(dark_image, unit=config.image_units)
# If we have overscan, subtract and trim.
if config.subtract_overscan:
if config_arguments.verbose_flag_2:
module_logger.info("Subtracting overscan of {0}".format(dark_image))
ccd = subtract_and_trim_overscan(ccd, config)
exposure_dark_list.append(ccd)
# median combine the data
cb = ccdproc.Combiner(exposure_dark_list)
master_dark = cb.median_combine(median_func=np.median)
# Add the masterdark to the master_flat collection
master_dark_collection.update({dark_exposure_item: master_dark})
# Save the masterdark if needed.
if config.save_masterdark:
# Filename to save
aux = '{0}/masterdark_{1}.fits'.format(config_arguments.save_path, dark_exposure_item)
module_logger.info('Saving master dark to {0}'.format(aux))
master_dark.to_hdu().writeto(aux)
# ------- MASTER FLAT CREATION --------
module_logger.info("Starting the creation of the master flats")
module_logger.info("{0} different filters for masterflats were found".format(
len(flat_filter_collection)))
master_flat_collection = dict()
# Go thought the different filters in the collection
for flat_filter in flat_filter_collection:
module_logger.info(
"Creating masterflat with filter {0}".format(flat_filter))
# Initializate the list that will carry the flat images of the actual
# filter
filter_flat_list = list()
for flat_image_data in filter_collection(flat_collection, [('filter', flat_filter)]):
# Open the images and append to the filter's flat list
flat_image = flat_image_data['filename']
ccd = CCDData.read(flat_image, unit=config.image_units)
# Subtract and trim overscan
if config.subtract_overscan:
if config_arguments.verbose_flag_2:
module_logger.info("Subtracting overscan of {0}".format(flat_image))
ccd = subtract_and_trim_overscan(ccd, config)
filter_flat_list.append(ccd)
# median combine the flats after scaling each by its mean
cb = ccdproc.Combiner(filter_flat_list)
cb.scaling = lambda x: 1.0 / np.mean(x)
master_flat = cb.median_combine(median_func=np.median)
# Add the masterflat to the master_flat collection
master_flat_collection.update({flat_filter: master_flat})
# Save the masterflat if needed.
if config.save_masterflat:
aux = '{0}/masterflat_{1}.fits'.format(config_arguments.save_path, flat_filter)
module_logger.info('Saving master flat to {0}'.format(aux))
master_flat.to_hdu().writeto(aux)
# ------- REDUCE SCIENCE DATA --------
module_logger.info("Starting the calibration of the science images")
# Go thought the different files in the collection
for image_filter in science_filter_collection:
module_logger.info("Now calibrating filter: {0}".format(image_filter))
# Iterate thought each different exposure. This is because the dark files
# can have different exposures and the calibration must be performed with
# the masterdark with the nearest exposure time.
for science_exposure in science_exposures_collection:
# Important!! If you have more classifiers in the numpy dtype and you want
# to use them, you must modify the code here. For example, if you want to
# use a 'temp' value as classifier, after modify the dtype following the
# instructions in FitsLookup, you must add a loop here and modify the sub_collection.
# Once you have the 'temp' in the dtype, you must add a loop here as:
#
# >>>for temp_value in set(science_collection['temp']):
# module_logger.info("Now calibrating temp: {0}".format(temp_value))
#
# After this, you MUST indent all the following code (of this function) four spaces to
# the right, of course. Then, you only have to modify the science_subcollection as follows:
#
# >>> science_subcollection = filter_collection(
# science_collection, [('filter', image_filter),
# ('exptime', science_exposure),
# ('temp', temp_value) ])
#
# Follow this steps for every classifier you want to add. Yay!
# --------------------------------------------------------------
# Science subcollection is a really bad name, but is descriptive. Remember that this subcollection
# are the images with the current filter that has the current
# exposure time. E.g. ('r' and 20', 'r' and 30).
science_subcollection = filter_collection(
science_collection, [('filter', image_filter),
('exptime', science_exposure)])
# Continue if we have files to process. This will check if for some filter
# there are not enought images with the actual exposure time.
if science_subcollection.size:
module_logger.info(
"Now calibrating exposure: {0}".format(science_exposure))
# Determine if we have a masterdark with the science exposure file.
#
# - If we have a exposure matching masterdark, use it.
# - If we do not have a exposure matching masterdark, use the nearest.
try:
selected_masterdark = master_dark_collection[science_exposure]
nearest_exposure = 0, science_exposure
except KeyError:
# Get the nearest exoposure in the dark collection.
nearest_exposure = min(enumerate(master_dark_collection.keys()),
key=lambda x: abs(x[1] - science_exposure))
# Notice that nearest_exposure is a tuple of the form
# (index,exposure).
selected_masterdark = master_dark_collection[
nearest_exposure[1]]
# Initialize the progress bar variables
total_len = len(science_subcollection)
meantime = []
# Loop for each image with current (filter,exptime).
for contador, science_image_data_with_current_exposure in enumerate(science_subcollection):
# To supress astropy warnings.
devnull = open(os.devnull, "w")
sys.stdout = devnull
# Notice that until sys stdout is reasigned, no printing
# will be allowed in the following lines.
# Start timing
start = time.time()
# Extract the filename from the image data
science_image = science_image_data_with_current_exposure['filename']
# Read the image
ccd = CCDData.read(science_image, unit=config.image_units, wcs=None)
# Subtract overscan
if config.subtract_overscan:
if config_arguments.verbose_flag_2:
module_logger.info("Subtracting overscan of {0}".format(science_image))
ccd = subtract_and_trim_overscan(ccd, config)
# Master dark substraction
if config_arguments.verbose_flag_2:
sys.stdout = sys.__stdout__ # Restart stdout printing
module_logger.info("Subtracting dark of image {0} of {1}".format(contador + 1, total_len))
sys.stdout = devnull
else:
module_logger.debug("Subtracting dark of image {0} of {1}".format(contador + 1, total_len))
selected_masterdark._wcs = ccd._wcs # FIXME: currently needed due to bug
ccd = ccdproc.subtract_dark(ccd, selected_masterdark, dark_exposure=nearest_exposure[1] * u.second,
data_exposure=science_exposure * u.second)
# flat-field correct the data
if config_arguments.verbose_flag_2:
sys.stdout = sys.__stdout__ # Restart stdout printing
module_logger.info("Flat-field correcting image {0} of {1}".format(contador + 1, total_len))
sys.stdout = devnull
else:
module_logger.debug("Flat-field correcting image {0} of {1}".format(contador + 1, total_len))
current_master_flat = master_flat_collection[image_filter]
current_master_flat._wcs = ccd._wcs # FIXME: currently needed due to bug
ccd = ccdproc.flat_correct(ccd, current_master_flat)
# If we need to clean cosmic rays, do it.
if config_arguments.cosmic_flag:
if config_arguments.verbose_flag_2:
sys.stdout = sys.__stdout__ # Restart stdout printing
module_logger.info(
"Cosmic ray cleaning of image {0} of {1}".format(contador + 1, total_len))
sys.stdout = devnull
else:
module_logger.debug(
"Cosmic ray cleaning of image {0} of {1}".format(contador + 1, total_len))
ccd = ccdproc.cosmicray_lacosmic(ccd, error_image=None, thresh=5, mbox=11, rbox=11, gbox=5)
# Save the calibrated image to a file
output_filename = os.path.join(config_arguments.save_path, os.path.basename(science_image))
if config_arguments.verbose_flag_2:
sys.stdout = sys.__stdout__ # Restart stdout printing
module_logger.info(
"Saving image {0} of {1} to {2}".format(contador + 1, total_len, output_filename))
sys.stdout = devnull
else:
module_logger.debug(
"Saving image {0} of {1} to {2}".format(contador + 1, total_len, output_filename))
ccd.write(output_filename, clobber=True)
end = time.time()
meantime.append(end - start)
sys.stdout = sys.__stdout__ # Restart stdout printing
# Progressbar in case that we have not activated the no_interaction flag nor the advance
# verbose flag.
if not config_arguments.no_interaction and not config_arguments.verbose_flag_2:
if config_arguments.verbose_flag:
update_progress(float(contador + 1) / total_len,
np.mean(meantime) * (total_len - (contador + 1)))
return 0
import ccdproc
from astropy.nddata import CCDData
import image_registration
read = ir_reduce.read_and_sort(glob.glob('../testdata/bad*.fits'),
glob.glob('../testdata/Flat*.fits'),
glob.glob('../testdata/NCA*.fits'))
processed = ir_reduce.tiled_process(read['J'].bad, read['J'].flat[0],
read['J'].images)
#processed = ir_reduce.standard_process(read['J'].bad, read['J'].flat[0], read['J'].images)
skyscaled = ir_reduce.skyscale(processed, 'subtract')
interpolated = [
ir_reduce.interpolate(image, dofixpix=True) for image in skyscaled
]
wcs = interpolated[0].wcs
reprojected = [ccdproc.wcs_project(img, wcs) for img in interpolated]
combined = ccdproc.Combiner(reprojected).median_combine()
#
# first_hdu = output_image.to_hdu()[0]
# scamp_input = CCDData(first_hdu.data, header=first_hdu.header, unit=first_hdu.header['bunit'])
#
# scamp_input = CCDData(first_hdu.data, header=first_hdu.header, unit=first_hdu.header['bunit'])
#
combined.wcs = wcs
combined.write('foo.fits', overwrite=True)