def bias_combine(refresh=0):
tbiascollection = ImageFileCollection('Trimmed_Bias')
print('found', len(tbiascollection.values('file')), 'trimmed biases')
combined_bias = ccdp.combine(tbiascollection.files_filtered(
imtype='trimmed bias', include_path=True),
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.nanmedian,
sigma_clip_dev_func=mad_std)
combined_bias.meta['combined'] = True
combined_bias.write('Master_Files/mbias.fits', overwrite=True)
return tbiascollection
def combine_flats(refresh='2', method='2'):
if method == '1':
meta = 'med'
source = 'Trimmed_Flat/subflatsmed'
dest = 'Master_Files/mflat_median.fits'
elif method == '2':
meta = 'sig'
source = 'Trimmed_Flat/subflatssig'
dest = 'Master_Files/mflat.fits'
subflatcollection = ImageFileCollection(source)
combtime = 0
if refresh == '1':
print('found', len(subflatcollection.values('file')), 'subflats')
start = time.time()
if method == '1':
mflat = ccdp.combine(subflatcollection.files_filtered(
imtype='subflat', include_path=True),
method='median')
mflat.meta['flatcom'] = 'median'
combtime = time.time() - start
print('combination took', combtime, 'seconds')
elif method == '2':
mflat = ccdp.combine(subflatcollection.files_filtered(
imtype='subflat', include_path=True),
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.nanmedian,
sigma_clip_dev_func=mad_std)
mflat.meta['flatcom'] = 'sigma'
combtime = time.time() - start
print('combination took', combtime, 'seconds')
mflat.meta['normmed'] = (np.nanmedian(mflat),
'nanmedian of the master flat')
mflat.meta['subflats'] = meta
mflat.write(dest[0:-5] + '_' + meta + '.fits', overwrite=True)
else:
try:
if method == '1':
mflat = CCDData.read('Master_Files/mflat_median_med.fits',
unit='adu')
elif method == '2':
mflat = CCDData.read('Master_Files/mflat_sig.fits', unit='adu')
except:
print('can\'t locate master flat, create or check directory')
sys.exit()
return subflatcollection, mflat, dest, combtime
def bias_combine(refresh='2', method='2'):
tbiascollection = ImageFileCollection('Trimmed_Bias')
combtime = 0
if refresh == '1':
print('found', len(tbiascollection.values('file')), 'trimmed biases')
start = time.time()
if method == '1':
combined_bias = ccdp.combine(tbiascollection.files_filtered(
imtype='trimmed bias', include_path=True),
method='median')
combbiaspath = 'Master_Files/mbias_median.fits'
combined_bias.meta['combined'] = 'median'
combtime = time.time() - start
print('combination took', combtime, 'seconds')
combined_bias.write(combbiaspath, overwrite=True)
elif method == '2':
combined_bias = ccdp.combine(tbiascollection.files_filtered(
imtype='trimmed bias', include_path=True),
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.nanmedian,
sigma_clip_dev_func=mad_std)
combbiaspath = 'Master_Files/mbias.fits'
combined_bias.meta['combined'] = 'sigma_clip average'
combtime = time.time() - start
print('combination took', combtime, 'seconds')
combined_bias.write(combbiaspath, overwrite=True)
else:
try:
if method == '1':
combined_bias = CCDData.read('Master_Files/mbias_median.fits',
unit='adu')
combbiaspath = 'Master_Files/mbias_median.fits'
elif method == '2':
combined_bias = CCDData.read('Master_Files/mbias.fits',
unit='adu')
combbiaspath = 'Master_Files/mbias.fits'
except:
print('can\'t locate master bias, create or check directory')
sys.exit()
return tbiascollection, combined_bias, combbiaspath, combtime
def bias_combine(refresh=False, method=1):
if refresh == 0:
tbiascollection = ImageFileCollection('Trimmed_Bias')
print('found', len(tbiascollection.values('file')), 'trimmed biases')
start=time.time()
if method == 1:
combined_bias = ccdp.combine(tbiascollection.files_filtered(
imtype='trimmed bias', include_path=True),
method='median')
combbiaspath = 'Master_Files/mbias_median.fits'
combined_bias.meta['combined'] = 'median'
elif method == 2:
combined_bias = ccdp.combine(tbiascollection.files_filtered(
imtype='trimmed bias', include_path=True),
sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5,
sigma_clip_func=np.nanmedian, sigma_clip_dev_func=mad_std)
combbiaspath = 'Master_Files/mbias.fits'
combined_bias.meta['combined'] = 'sigma_clip average'
combtime=time.time()-start
print('combination took',combtime,'seconds')
combined_bias.write('Master_Files/mbias_median.fits',
overwrite=True)
return tbiascollection,combined_bias,combbiaspath,combtime
import glob
from astropy import units as u
from astropy.io import fits
from ccdproc import CCDData, Combiner
import numpy as np
from ccdproc import Combiner, ImageFileCollection, fits_ccddata_writer
from t120_init import t120
listimg = ImageFileCollection(
t120.t120_flat_dir) #,glob_include='*.fit',glob_exclude='*.fits')
list_filters = listimg.values('filter', unique=True)
#for filter_name in listimg.values('filter',unique=True):
for filter_name in list_filters:
t120.log.info('*** filter: ' + filter_name)
my_files = listimg.files_filtered(filter=filter_name)
t120.log.info('my_files=' + my_files)
pouet
for filter_name in listimg.values('filter', unique=True):
t120.log.info('*** filter: ' + filter_name)
listccd = []
for ccd, file_name in listimg.ccds(ccd_kwargs={'unit': 'adu'},
filter=filter_name,
return_fname=True):
t120.log.info('now considering file ' + file_name)
listccd.append(ccd)
"""
t120.log.info('now making the Combiner object')
combiner = Combiner(listccd)
print('not combining zeros')
print('\t reading in bias-combined.fits instead')
hdu1 = fits.open('bias-combined.fits')
header = hdu1[0].header
gaincorrected_master_bias = CCDData(hdu1[0].data, unit=u.electron, meta=header)
hdu1.close()
else:
print('not using a bias frame')
###################################################
# COMBINE DARKS
###################################################
# identify darks and combine darks with longest exposure time
# change to use the dark with the closest exposure time
exptimes = np.array(ic.values('exptime'))
image_types = np.array(ic.values('imagetyp'))
if args.darkcombine:
# select all files with imagetyp=='dark'
# want to read in one set of long exposure dark frames, like 120 s
# observers should take a set of darks that correspond to longest exposure time
# e.g. 120s
set_exptime=set(exptimes[image_types == ccdkeyword['dark']])
dark_exptimes_set = np.array(list(set_exptime),'f')
max_exposure = max(exptimes[image_types == ccdkeyword['dark']])
'''
* subtract bias from combined dark frames
def na_back_pipeline(directory=None,
glob_include='Jupiter*',
calibration=None,
photometry=None,
n_back_boxes=N_BACK_BOXES,
num_processes=None,
outdir=None,
create_outdir=True,
**kwargs):
outdir = outdir or reduced_dir(directory, create=False)
collection = ImageFileCollection(directory, glob_include=glob_include)
if collection is None:
return []
summary_table = collection.summary
#print(summary_table['raoff'])
try:
raoffs = collection.values('raoff', unique=True)
decoffs = collection.values('decoff', unique=True)
except Exception as e:
log.error(f'Problem with RAOFF/DECOFF in {directory}: {e}')
return []
f_pairs = []
for raoff in raoffs:
for decoff in decoffs:
try:
subc = collection.filter(raoff=raoff, decoff=decoff)
except:
log.debug(f'No match for RAOFF = {raoff} DECOFF = {decoff}')
continue
fp = closest_in_time(subc, ('Na_on', 'Na_off'),
valid_long_exposure,
directory=directory)
f_pairs.extend(fp)
if len(f_pairs) == 0:
log.warning(f'No matching set of Na background files found '
f'in {directory}')
return []
if calibration is None:
calibration = Calibration(reduce=True)
if photometry is None:
photometry = Photometry(precalc=True,
n_back_boxes=n_back_boxes,
**kwargs)
# Eventually put back in
cmp = CorMultiPipe(auto=True,
calibration=calibration,
photometry=photometry,
create_outdir=create_outdir,
post_process_list=[
multi_filter_proc, combine_masks, na_back_process,
no_outfile
],
num_processes=num_processes,
process_expand_factor=15,
**kwargs)
# but get ready to write to reduced directory if necessary
#pout = cmp.pipeline([f_pairs[0]], outdir=outdir, overwrite=True)
pout = cmp.pipeline(f_pairs, outdir=outdir, overwrite=True)
pout, _ = prune_pout(pout, f_pairs)
return pout
def go_swarp(inputfiles,
output_filename = None,
configfile = None,
verbose = True):
"""Do SWARP"""
# Make sure we have a configuration file:
if configfile == None:
# TODO: replace this with default config file in LBCgo directories.
configfile = 'swarp.lbc.conf'
# Gather some information about the input files
keywds = ['object', 'filter', 'exptime', 'imagetyp', 'propid', 'lbcobnam',
'airmass', 'HA', 'objra', 'objdec']
ic_swarp = ImageFileCollection('./', keywords=keywds,
filenames = inputfiles)
# Check all are the same filter:
# TODO: Set exception if not all are same filter rather than just bail.
fltrs = ic_swarp.values('filter',unique=True)
if np.size(fltrs) != 1:
print('Warning: not all files in SWARP call use the same filter.')
return None
# Calculate mean airmass (weighted by exposure time)
exp_airmass = np.array(ic_swarp.values('airmass'))
exp_time = np.array(ic_swarp.values('exptime'))
airmass = np.average(exp_airmass,weights=exp_time)
# from IPython import embed ; embed()
# For now grab the information from the first header:
if output_filename == None:
imhead = fits.getheader(inputfiles[0])
# Shorten the filter names used:
filter_text = imhead['FILTER']
filter_text = filter_text.replace('-SLOAN','').replace('-BESSEL','').replace('SDT_Uspec','Uspec')
# Create final output filename
# TODO: Check if file exists.
output_filename = (imhead['object']).\
replace(' ','')+'.'+filter_text+'.mos.fits'
# Rename the weight image
weight_filename = output_filename.replace('.mos.fits','.mos.weight.fits')
# Create the list of input files:
inputfile_text = ''
for fl in inputfiles: inputfile_text = inputfile_text+' '+fl
cmd_flags = ' -c '+configfile+ \
' -IMAGEOUT_NAME '+ output_filename + \
' -WEIGHTOUT_NAME '+ weight_filename + \
' -WEIGHT_TYPE NONE '+ \
' -HEADER_SUFFIX ".head"'+ \
' -FSCALE_KEYWORD NONE -FSCALE_DEFAULT 1.0 '+\
' -CELESTIAL_TYPE EQUATORIAL -CENTER_TYPE ALL '+\
' -COMBINE_BUFSIZE 4096 ' +\
' -COPY_KEYWORDS '+\
' OBJECT,OBJRA,OBJDEC,OBJEPOCH,PROPID,PI_NAME,'+\
'FILTER,SATURATE,RDNOISE,GAIN,EXPTIME,AIRMASS,TIME-OBS'
# Create the final command:
cmd = 'swarp ' + inputfile_text + cmd_flags
try:
if verbose:
print('########### SWARP image combination '
'########### \n')
print(cmd)
swarp = Popen(shlex.split(cmd),
close_fds=True)
else:
swarp = Popen(shlex.split(cmd),
stdout=DEVNULL,
stderr=DEVNULL,
close_fds=True)
except Exception as e:
print('Oops: SWARP call:', (e))
return None
swarp.wait()
# Add airmass to header:
fits.setval(output_filename,'AIRMASS',value=airmass)
def lbcgo(raw_directory='./raw/',
image_directory='./',
lbc_chips=True,
lbcr=True,
lbcb=True,
filter_names=None,
bias_proc=False,
do_astrometry=True,
scamp_iterations=3,
verbose=True,
clean=True):
"""Process a directory of LBC data.
By default, all raw data are to be in the ./raw/ directory before processing (raw_directory='./raw/').
By default, all new images are written in the CWD (image_directory='./').
"""
# It will go something like this...
# Make sure the input directories have trailing slashes:
if raw_directory[-1] != '/':
raw_directory += '/'
if image_directory[-1] != '/':
image_directory += '/'
########## Collect basic image information
# Find the raw files we'll use
if lbcb & lbcr:
lbc_file_base = 'lbc?.*.*.fits*'
elif lbcb:
lbc_file_base = 'lbcb.*.*.fits*'
elif lbcr:
lbc_file_base = 'lbcr.*.*.fits*'
# What information do we want from the headers?
keywds = [
'object', 'filter', 'exptime', 'imagetyp', 'propid', 'lbcobnam',
'airmass', 'HA', 'objra', 'objdec'
]
##### Create an ImageFileCollection object to hold the raw data list.
if np.int(ccdproc.__version__[0]) == 2:
ic0 = ImageFileCollection(raw_directory,
keywords=keywds,
glob_include=lbc_file_base)
num_images = np.size(ic0.summary['object'])
# Exit if (for some reason) no images are found in the raw_directory.
if num_images == 0:
print('WARNING: No images found.')
return None
else:
raw_lbc_files = glob(raw_directory + lbc_file_base)
ic0 = ImageFileCollection(raw_directory,
keywords=keywds,
filenames=raw_lbc_files)
num_images = np.size(ic0.summary['object'])
# Exit if (for some reason) no images are found in the raw_directory.
if num_images == 0:
print('WARNING: No images found.')
return None
######### Create the master bias frame (if requested)
if bias_proc == True:
make_bias(ic0,
image_directory=image_directory,
raw_directory=raw_directory)
###### Define which chips to extract if default is chosen:
if lbc_chips == True:
lbc_chips = [1, 2, 3, 4]
###### Per filter:
#
# Step through the filters in the ImageCollection unless user gives filter
# list.
#
# For now nights with V-band filters in both LBCB and LBCR require running
# the code with 'lbcr=False' then 'lbcb=False' to avoid coadding the two
# images.
# TODO: Loop B/R to avoid issues w/V-band from LBCB/LBCR?
# TODO: Check for co-pointing files
if filter_names == None:
icX = ImageFileCollection(
raw_directory,
keywords=keywds,
filenames=(ic0.files_filtered(imagetyp='object')).tolist())
filter_names = icX.values('filter', unique=True)
# Loop through each of the filters
for filter in filter_names:
# Find the images with the filter of interest.
if verbose == True:
print('Processing {0} files.'.format(filter))
# List of images in the current filter
ic1 = ImageFileCollection(
raw_directory,
keywords=keywds,
filenames=(ic0.files_filtered(filter=filter)).tolist())
# Make master flat fields.
# Check to see if flat exists; if so, skip making the flat (time
# consuming)
flatname = 'flat.' + filter + '.fits'
if not os.path.lexists(flatname):
make_flatfield(ic1,
verbose=verbose,
raw_directory=raw_directory,
image_directory=image_directory)
else:
print('Using existing {0}'.format(flatname))
# Remove overscan, trim object files.
overfiles = go_overscan(ic1,
lbc_chips=lbc_chips,
verbose=verbose,
raw_directory=raw_directory,
image_directory=image_directory)
# Apply bias frames
if bias_proc == True:
# zero_files = go_bias(verbose=verbose)
print('')
# Image collection of object frames overscanned & bias subtracted
ic2 = ImageFileCollection(image_directory,
keywords=keywds,
filenames=overfiles)
# Apply the flatfields
flatfiles = go_flatfield(ic2,
return_files=True,
image_directory=image_directory,
verbose=verbose,
cosmiccorrect=False)
# Image collection of object frames overscanned & bias subtracted + flattened
ic3 = ImageFileCollection(image_directory,
keywords=keywds,
filenames=flatfiles)
# from IPython import embed ; embed()
# Create directories for extracting individual chips.
tgt_dirs, fltr_dirs = make_targetdirectories(
ic3, image_directory=image_directory, verbose=verbose)
# Extract individual chips
go_extractchips(fltr_dirs, lbc_chips, verbose=verbose)
# Register and coadd the images
if do_astrometry:
go_register(fltr_dirs,
lbc_chips=lbc_chips,
scamp_iterations=scamp_iterations)
# Let's do some clean-up.
if clean:
# We will eventually ...
#remove _over, _zero, _flat files.
print('')
else:
# We will eventually ...
# Move _over, _zero, _flat files.
print('')
def make_targetdirectories(image_collection,
image_directory='./',
object_names=None,
verbose=True):
""" Make directories to store data from individual targets and move the data for those targets into the directories.
:param image_collection:
:param image_directory:
:param object_names:
:param verbose:
:return: object_directories, filter_directories
"""
# If we specify objects, use only those objects
if object_names == None:
object_names = np.unique(image_collection.summary['object'])
# Return a list of the directories
object_directories = []
filter_directories = []
# Loop through the objects
for obj in object_names:
# Name of the target directory to be created
dirname = image_directory + obj.replace(' ', '') + '/'
object_directories.append(dirname)
# Test that the directory doesn't already exist
if not os.path.lexists(dirname):
os.makedirs(dirname)
if verbose == True:
print('Creating directory for object {0}.'.format(obj))
else:
if verbose == True:
print('Directory exists for object {0}.'.format(obj))
# Move the data for each object into its directory.
# object_files = image_collection.files_filtered(object=obj)
object_files = \
image_collection.summary['file'][(image_collection.summary['object'] == obj)]
for fl in object_files:
cmd = 'mv {0} {1}'.format(fl, dirname)
crap = call(cmd, shell=True)
# Create filter-specific directories and fill them. For now this just
# assumes that every object has the same filter list. This is fine,
# since we're normally going through this step on a filter-by-filter
# basis anyway (see proc.py).
# The following should select only the filters appropriate for this object:
keywds = ['object', 'filter']
image_collectionObj = ImageFileCollection(dirname, keywords=keywds,
filenames = \
(image_collection.files_filtered(object=obj.replace('+','\+').replace('-','\-'))).tolist())
# debug
# from IPython import embed ; embed()
# Now select the unique filters for this objects
filters = image_collectionObj.values('filter', unique=True)
# Step through each filter, creating directories and moving files.
for filter in filters:
filter_dirname = dirname + filter + '/'
filter_directories.append(filter_dirname)
# Test that the directory doesn't already exist
if not os.path.lexists(filter_dirname):
os.makedirs(filter_dirname)
if verbose == True:
print('Creating {1} directory for object {0}.'.format(
obj, filter))
else:
if verbose == True:
print('Directory exists for {1} for object {0}.'.format(
obj, filter))
filter_files = image_collection.files_filtered(object=obj.replace(
'+', '\+').replace('-', '\-'),
filter=filter)
for fltfl in filter_files:
cmd = 'mv {0} {1}'.format(dirname + fltfl, filter_dirname)
print(cmd)
crap = call(cmd, shell=True)
return object_directories, filter_directories