def combine_flats(refresh='2', method='2'):
if method == '1':
meta = 'med'
source = 'Trimmed_Flat/subflatsmed'
dest = 'Master_Files/mflat_median.fits'
elif method == '2':
meta = 'sig'
source = 'Trimmed_Flat/subflatssig'
dest = 'Master_Files/mflat.fits'
subflatcollection = ImageFileCollection(source)
combtime = 0
if refresh == '1':
print('found', len(subflatcollection.values('file')), 'subflats')
start = time.time()
if method == '1':
mflat = ccdp.combine(subflatcollection.files_filtered(
imtype='subflat', include_path=True),
method='median')
mflat.meta['flatcom'] = 'median'
combtime = time.time() - start
print('combination took', combtime, 'seconds')
elif method == '2':
mflat = ccdp.combine(subflatcollection.files_filtered(
imtype='subflat', include_path=True),
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.nanmedian,
sigma_clip_dev_func=mad_std)
mflat.meta['flatcom'] = 'sigma'
combtime = time.time() - start
print('combination took', combtime, 'seconds')
mflat.meta['normmed'] = (np.nanmedian(mflat),
'nanmedian of the master flat')
mflat.meta['subflats'] = meta
mflat.write(dest[0:-5] + '_' + meta + '.fits', overwrite=True)
else:
try:
if method == '1':
mflat = CCDData.read('Master_Files/mflat_median_med.fits',
unit='adu')
elif method == '2':
mflat = CCDData.read('Master_Files/mflat_sig.fits', unit='adu')
except:
print('can\'t locate master flat, create or check directory')
sys.exit()
return subflatcollection, mflat, dest, combtime
def get_files(filenames,
location='.',
imagetyp=None,
filter=None,
fexptime='*'):
'''Gather files from input directories, choosing by image type filter
if specified'''
# for summary display only
keywords = ('object', 'date-obs', 'IMAGETYP', 'FILTER', 'EXPTIME')
collection = ImageFileCollection(filenames=filenames,
location=location,
keywords=keywords)
if imagetyp in ['light', 'bias', 'dark']:
imagetyp = '%s frame' % imagetyp
else:
imagetyp = '*'
if fexptime is None:
fexptime = '*'
if imagetyp == 'bias frame':
fexptime = 0.
# filter and reread
if filter is None:
filelist = collection.files_filtered(IMAGETYP=imagetyp,
EXPTIME=fexptime,
include_path=True)
else:
filelist = collection.files_filtered(IMAGETYP=imagetyp,
EXPTIME=fexptime,
FILTER=filter,
include_path=True)
if not filelist:
raise RuntimeError('No matching files found.')
location = os.path.join(collection.location, os.path.dirname(filenames[0]))
collection = ImageFileCollection(filenames=filelist,
location=location,
keywords=keywords)
collection.summary.pprint()
return collection
def sub_bias(refresh='2', bias='2'):
tflatcollection = ImageFileCollection('Trimmed_Flat')
if bias == '1':
biaspath = 'Master_Files/mbias_median.fits'
dest = 'Trimmed_Flat/subflatsmed/'
elif bias == '2':
biaspath = 'Master_Files/mbias.fits'
dest = 'Trimmed_Flat/subflatssig/'
if refresh == '1':
subflatpathlist = []
mbias = CCDData.read(biaspath, unit='adu')
for ccdf, flatn in tflatcollection.ccds(imtype='trimmed flat',
return_fname=True):
subflat = ccdp.subtract_bias(ccdf, mbias, add_keyword='subbias')
subflat.meta['imtype'] = ('subflat', 'bias subtracted flat')
subflat.write(dest + flatn[0:8] + '_subbias.fits', overwrite=True)
subflatpathlist.append(dest + flatn[0:8] + '_subbias.fits')
else:
try:
subflatcollection = ImageFileCollection(dest)
subflatpathlist = subflatcollection.files_filtered(
imtype='subflat', include_path=True)
print('found', len(subflatpathlist), 'subflats')
except:
print('can\'t locate subflats, create or check directory')
sys.exit()
return tflatcollection, subflatpathlist
def trim_bias(refresh='2'):
biascollection = ImageFileCollection('HD115709/bias', ext=4)
flag = 0
if refresh == '1':
tbiaspathlist = []
for ccdb, biasn in biascollection.ccds(return_fname=True,
ccd_kwargs={'unit': 'adu'}):
if flag == 0:
print('all biases will be trimmed to :', ccdb.meta['trimsec'])
flag = 1
print('trimming', biasn)
tbias = ccdp.trim_image(ccdb,
fits_section=str(ccdb.meta['trimsec']))
tbias.meta['imtype'] = ('trimmed bias', 'type of image')
tbias.meta['taxis1'] = (2048, 'dimension1')
tbias.meta['taxis2'] = (4096, 'dimension2')
tbias.write('Trimmed_Bias/' + biasn[0:8] + '_trim.fits',
overwrite=True)
tbiaspathlist.append('Trimmed_Bias/' + biasn[0:8] + '_trim.fits')
print('created', len(tbiaspathlist), 'trimmed biases')
else:
try:
tbiascollection = ImageFileCollection('Trimmed_Bias')
tbiaspathlist = tbiascollection.files_filtered(
imtype='trimmed bias', include_path=True)
print('found', len(tbiaspathlist), 'trimmed bias')
except:
print('can\'t locate trimmed biases, create or check directory')
sys.exit()
return biascollection, tbiaspathlist
def trim_flat(refresh='2'):
flatcollection = ImageFileCollection('HD115709/flat_SII', ext=4)
flag = 0
tflatpathlist = []
if refresh == '1':
for ccdf, flatn in flatcollection.ccds(return_fname=True,
ccd_kwargs={'unit': 'adu'}):
if flag == 0:
print('all flats will be trimmed to :', ccdf.meta['trimsec'])
flag = 1
print('trimming', flatn)
tflat = ccdp.trim_image(ccdf,
fits_section=str(ccdf.meta['trimsec']))
tflat.meta['imtype'] = ('trimmed flat', 'type of image')
tflat.meta['taxis1'] = (2048, 'dimension1')
tflat.meta['taxis2'] = (4096, 'dimension2')
tflat.write('Trimmed_Flat/' + flatn[0:8] + '_trim.fits',
overwrite=True)
tflatpathlist.append('Trimmed_Flat/' + flatn[0:8] + '_trim.fits')
print('created', len(tflatpathlist), 'trimmed flats')
elif refresh == '2':
try:
tflatcollection = ImageFileCollection('Trimmed_Flat')
tflatpathlist = tflatcollection.files_filtered(
imtype='trimmed flat', include_path=True)
print('found', len(tflatpathlist), 'trimmed flats')
except:
print('can\'t locate trimmed flats, create or check directory')
sys.exit(0)
return flatcollection, tflatpathlist
def fix_headers(folder):
"""
Find FITS files in folder that have WCS CTYPE ``RA--TAN`` and ``DEC--TAN``
and add ``-SIP`` to the end. Necessary because astropy handling of SIP
distortion keywords changed in v1.2.
Parameters
----------
folder : str
Path to folder with the images to fix.
"""
ic = ImageFileCollection(folder)
ic.summary['ctype1']
files = ic.files_filtered(ctype1='RA---TAN')
if not len(files):
print('No files to fix in {}'.format(folder))
for file in files:
fname = os.path.join(folder, file)
print('Fixing file {}'.format(file))
with fits.open(fname) as f:
f[0].header['ctype1'] = f[0].header['ctype1'] + '-SIP'
f[0].header['ctype2'] = f[0].header['ctype2'] + '-SIP'
f.writeto(fname, clobber=True)
def bias_combine(refresh='2', method='2'):
tbiascollection = ImageFileCollection('Trimmed_Bias')
combtime = 0
if refresh == '1':
print('found', len(tbiascollection.values('file')), 'trimmed biases')
start = time.time()
if method == '1':
combined_bias = ccdp.combine(tbiascollection.files_filtered(
imtype='trimmed bias', include_path=True),
method='median')
combbiaspath = 'Master_Files/mbias_median.fits'
combined_bias.meta['combined'] = 'median'
combtime = time.time() - start
print('combination took', combtime, 'seconds')
combined_bias.write(combbiaspath, overwrite=True)
elif method == '2':
combined_bias = ccdp.combine(tbiascollection.files_filtered(
imtype='trimmed bias', include_path=True),
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.nanmedian,
sigma_clip_dev_func=mad_std)
combbiaspath = 'Master_Files/mbias.fits'
combined_bias.meta['combined'] = 'sigma_clip average'
combtime = time.time() - start
print('combination took', combtime, 'seconds')
combined_bias.write(combbiaspath, overwrite=True)
else:
try:
if method == '1':
combined_bias = CCDData.read('Master_Files/mbias_median.fits',
unit='adu')
combbiaspath = 'Master_Files/mbias_median.fits'
elif method == '2':
combined_bias = CCDData.read('Master_Files/mbias.fits',
unit='adu')
combbiaspath = 'Master_Files/mbias.fits'
except:
print('can\'t locate master bias, create or check directory')
sys.exit()
return tbiascollection, combined_bias, combbiaspath, combtime
def bias_combine(refresh=0):
tbiascollection = ImageFileCollection('Trimmed_Bias')
print('found', len(tbiascollection.values('file')), 'trimmed biases')
combined_bias = ccdp.combine(tbiascollection.files_filtered(
imtype='trimmed bias', include_path=True),
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.nanmedian,
sigma_clip_dev_func=mad_std)
combined_bias.meta['combined'] = True
combined_bias.write('Master_Files/mbias.fits', overwrite=True)
return tbiascollection
def bias_combine(refresh=False, method=1):
if refresh == 0:
tbiascollection = ImageFileCollection('Trimmed_Bias')
print('found', len(tbiascollection.values('file')), 'trimmed biases')
start=time.time()
if method == 1:
combined_bias = ccdp.combine(tbiascollection.files_filtered(
imtype='trimmed bias', include_path=True),
method='median')
combbiaspath = 'Master_Files/mbias_median.fits'
combined_bias.meta['combined'] = 'median'
elif method == 2:
combined_bias = ccdp.combine(tbiascollection.files_filtered(
imtype='trimmed bias', include_path=True),
sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5,
sigma_clip_func=np.nanmedian, sigma_clip_dev_func=mad_std)
combbiaspath = 'Master_Files/mbias.fits'
combined_bias.meta['combined'] = 'sigma_clip average'
combtime=time.time()-start
print('combination took',combtime,'seconds')
combined_bias.write('Master_Files/mbias_median.fits',
overwrite=True)
return tbiascollection,combined_bias,combbiaspath,combtime
def get_files(filenames,location='.',imagetyp=None,filter=None,fexptime='*'):
'''Gather files from input directories, choosing by image type filter
if specified'''
# for summary display only
keywords = ('object','date-obs','IMAGETYP','FILTER','EXPTIME')
collection = ImageFileCollection(filenames=filenames,location=location,
keywords=keywords)
if imagetyp in ['light','bias','dark']:
imagetyp = '%s frame' % imagetyp
else:
imagetyp = '*'
if fexptime is None:
fexptime = '*'
if imagetyp == 'bias frame':
fexptime = 0.
# filter and reread
if filter is None:
filelist = collection.files_filtered(IMAGETYP=imagetyp,EXPTIME=fexptime,include_path=True)
else:
filelist = collection.files_filtered(IMAGETYP=imagetyp,EXPTIME=fexptime,FILTER=filter,include_path=True)
if not filelist:
raise RuntimeError('No matching files found.')
location = os.path.join(collection.location,os.path.dirname(filenames[0]))
collection = ImageFileCollection(filenames=filelist,location=location,
keywords=keywords)
collection.summary.pprint()
return collection
def load_files(target_dir, config, files, logger):
from ccdproc import ImageFileCollection
keywords = [
'object', 'imagetyp', 'date-obs', 'telra', 'teldec', 'airmass',
'filter', 'oairtemp', 'relhum', 'subbias', 'flatcor'
]
ic = ImageFileCollection(location=target_dir,
filenames=files,
keywords=keywords)
nbias = len(ic.files_filtered(imagetyp='bias'))
nflat = len(ic.files_filtered(imagetyp='flat'))
ndata = len(ic.files_filtered(imagetyp='object'))
filters = []
for h in ic.headers():
if h['IMAGETYP'] in ['object', 'flat']:
if h['FILTER'] not in filters:
filters.append(h['FILTER'])
filters.sort()
# flat_breakdown = []
# data_breakdown = []
# for filt in filters:
# for k in config.flats.values():
# flat_breakdown.append('{} {} {}'.format(
# len(ic.files_filtered(
# imagetyp='flat', object=k, filter=filt)),
# filt, k))
# data_breakdown.append('{} {}'.format(
# len(ic.files_filtered(imagetyp='OBJECT', filter=filt)), filt))
logger.info('{} files: {} bias, {} flats, {} object ({} filters)'.format(
len(ic.files), nbias, nflat, ndata, len(filters)))
return ic
def test_run_astrometry_with_dest_does_not_modify_source(self):
destination = self.test_dir.make_numbered_dir()
list_before = self.test_dir.listdir(sort=True)
arglist = [
'--destination-dir', destination.strpath, self.test_dir.strpath
]
run_astrometry.main(arglist)
list_after = self.test_dir.listdir(sort=True)
# nothing should change in the source directory
assert (list_before == list_after)
# for each light file in the destination directory we should have a
# file with the same basename but an extension of blind
ic = ImageFileCollection(destination.strpath, keywords=['IMAGETYP'])
for image in ic.files_filtered(imagetyp='LIGHT'):
image_path = destination.join(image)
print(image_path.purebasename)
blind_path = destination.join(image_path.purebasename + '.blind')
print(blind_path.strpath)
assert (blind_path.check())
ic1 = ImageFileCollection(indir.replace('sci2', '20160115'))
#create an array of dates
arm = 'Red arm'
file_list = []
date_list=[]
for hdu, fname in ic1.hdus(obstype='Arc', isiarm=arm, return_fname=True):
if os.path.isfile('w_arc_'+os.path.basename(fname)):
d = hdu.header['DATE-OBS'] + ' ' + hdu.header['UT']
d = datetime.strptime(d, '%Y-%m-%d %H:%M:%S.%f')
file_list.append(fname)
date_list.append(d)
date_arr = np.array(date_list)
#reduce the object frames
for filename in ic1.files_filtered(obstype='TARGET', isiarm=arm):
hdu = fits.open('obj_' + filename)
#find the closest two arcs in time
#open them, based on the time difference
#average their frames, and then use that as the
#wave map for the object file
date = hdu[0].header['DATE-OBS'] + ' ' + hdu[0].header['UT']
date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S.%f')
d = abs(date_arr - date)
i = d.argmin()
if date_arr[i] < date:
j = i+1
else:
j = i-1
import glob
from astropy import units as u
from astropy.io import fits
from ccdproc import CCDData, Combiner
import numpy as np
from ccdproc import Combiner, ImageFileCollection, fits_ccddata_writer
from t120_init import t120
listimg = ImageFileCollection(
t120.t120_flat_dir) #,glob_include='*.fit',glob_exclude='*.fits')
list_filters = listimg.values('filter', unique=True)
#for filter_name in listimg.values('filter',unique=True):
for filter_name in list_filters:
t120.log.info('*** filter: ' + filter_name)
my_files = listimg.files_filtered(filter=filter_name)
t120.log.info('my_files=' + my_files)
pouet
for filter_name in listimg.values('filter', unique=True):
t120.log.info('*** filter: ' + filter_name)
listccd = []
for ccd, file_name in listimg.ccds(ccd_kwargs={'unit': 'adu'},
filter=filter_name,
return_fname=True):
t120.log.info('now considering file ' + file_name)
listccd.append(ccd)
"""
t120.log.info('now making the Combiner object')
combiner = Combiner(listccd)
header['HISTORY'] = '= Cosmic rays rejected using ccdproc.cosmicray_lacosmic '
fits.writeto('z'+f,crimage,header)
hdu1.close()
i += 1
print ('\n')
# make an image collection of all the files in the data directory
# z is prefix for galaxies that already have cosmic ray rejection
ic = ImageFileCollection(os.getcwd(),keywords='*',glob_include='*.fit*')
# combine bias frames
if args.zerocombine:
# select all files with imagetyp=='bias'
bias_files = ic.files_filtered(imagetyp = ccdkeyword['bias'])
# feed list into ccdproc.combine, output bias
master_bias = ccdproc.combine(bias_files,method='average',sigma_clip=True,unit=u.adu)
gaincorrected_master_bias = ccdproc.gain_correct(master_bias,float(gain))
print('writing fits file for master bias')
gaincorrected_master_bias.write('bias-combined.fits',overwrite=True)
else:
if args.bias:
print('not combining zeros')
print('\t reading in bias-combined.fits instead')
hdu1 = fits.open('bias-combined.fits')
header = hdu1[0].header
gaincorrected_master_bias = CCDData(hdu1[0].data, unit=u.electron, meta=header)
hdu1.close()
else:
if len(sys.argv)!=3:
print('Usage:\npython wht_basic_rection.py [full_path_to_raw_data] [full_path_to_reduced_data]\n')
exit()
indir = sys.argv[1]
outdir = sys.argv[2]
if not os.path.isdir(outdir): os.mkdir(outdir)
os.chdir(outdir)
#change this to point to your raw data directory
ic1 = ImageFileCollection(indir)
#create the bias frames
blue_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Blue arm'):
print ic1.location + filename
ccd = CCDData.read(ic1.location + filename, unit = u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
blue_bias_list.append(ccd)
master_bias_blue = ccdproc.combine(blue_bias_list, method='median')
master_bias_blue.write('master_bias_blue.fits', clobber=True)
red_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Red arm'):
print ic1.location + filename
ccd = CCDData.read(ic1.location + filename, unit = u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
'Usage:\npython wht_basic_rection.py [full_path_to_raw_data] [full_path_to_reduced_data]\n'
)
exit()
indir = sys.argv[1]
outdir = sys.argv[2]
if not os.path.isdir(outdir): os.mkdir(outdir)
os.chdir(outdir)
#change this to point to your raw data directory
ic1 = ImageFileCollection(indir)
#create the bias frames
blue_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Blue arm'):
print ic1.location + filename
ccd = CCDData.read(ic1.location + filename, unit=u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd,
median=True,
overscan_axis=0,
fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'])
blue_bias_list.append(ccd)
master_bias_blue = ccdproc.combine(blue_bias_list, method='median')
master_bias_blue.write('master_bias_blue.fits', clobber=True)
red_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Red arm'):
print ic1.location + filename
def find_files(path, db, config, logger):
files = glob(os.sep.join((path, config.file_template)))
files = sorted([os.path.basename(f) for f in files])
keywords = [
'object', 'imagetyp', 'date-obs', 'ut', 'telra', 'teldec', 'airmass',
'filter', 'exptime', 'oairtemp', 'relhum', 'seeing', 'subbias',
'flatcor'
]
ic = ImageFileCollection(location=path, filenames=files, keywords=keywords)
files = sorted(list(ic.files_filtered(imagetyp='object')))
logger.debug('Found {} object files in {}.'.format(len(files), path))
if len(files) == 0:
raise ObjectFilesError('No object files found in {}.'.format(path))
ic = ImageFileCollection(location=path, filenames=files, keywords=keywords)
rows = []
for row in ic.summary:
fn = row['file'].replace('.fits', '')
c = db.execute('SELECT * FROM obs WHERE filename=?', [fn])
if c.fetchone() is None or config.update_obs:
with fits.open(os.sep.join((path, row['file']))) as hdu:
wcs = WCS(hdu[0])
shape = hdu[0].data.shape
if ((wcs.wcs.crpix[0] == 0) or (wcs.wcs.crpix[0] == 1)
or ('PIXEL' in wcs.wcs.ctype)):
# missing WCS solution
ra, dec = '0', '0'
else:
ra, dec = wcs.all_pix2world(shape[1] / 2, shape[0] / 2, 0)
c = SkyCoord(ra, dec, unit='deg')
ra, dec = c.to_string('hmsdms', sep=':').split()
rows.append(
(fn, row['object'], row['date-obs'] + ' ' + row['ut'],
row['airmass'], row['filter'], row['exptime'], ra, dec,
row['oairtemp'], row['relhum'], row['seeing'] / 2))
if len(rows) > 0:
if config.update_obs:
cols = list(zip(*rows))
update = zip(*(cols + [cols[0]]))
logger.info('Updating with FITS header meta data.')
db.executemany(
'''
UPDATE OR IGNORE obs SET
filename=?,object=?,obsdate=?,airmass=?,filter=?,exptime=?,
ra=?,dec=?,oairtemp=?,relhum=?,seeing=?
WHERE filename=?
''', update)
# insert whatever wasn't updated
db.executemany(
'''
INSERT OR IGNORE INTO obs
(filename,object,obsdate,airmass,filter,exptime,ra,dec,
oairtemp,relhum,seeing)
VALUES
(?,?,?,?,?,?,?,?,?,?,?)''', rows)
return ic
def triage_fits_files(dir=None, file_info_to_keep=None):
"""
Check FITS files in a directory for deficient headers
`dir` is the name of the directory to search for files.
`file_info_to_keep` is a list of the FITS keywords to get values
for for each FITS file in `dir`.
"""
dir = dir or '.'
all_file_info = file_info_to_keep or [
'imagetyp', 'object', 'filter', 'wcsaxes'
]
feder = Feder()
RA = feder.RA
if ((not (set(RA.names) <= set(all_file_info)))
and (all_file_info != '*')):
all_file_info.extend(RA.names)
images = ImageFileCollection(dir, keywords=all_file_info)
file_info = images.summary
# check for bad image type and halt until that is fixed.
if contains_maximdl_imagetype(images):
raise ValueError(
'Correct MaxImDL-style image types before proceeding.')
file_needs_filter = \
list(images.files_filtered(imagetyp='light',
filter=None))
file_needs_filter += \
list(images.files_filtered(imagetyp='flat',
filter=None))
file_needs_object_name = \
list(images.files_filtered(imagetyp='light',
object=None))
lights = file_info[file_info['imagetyp'] == 'LIGHT']
file_needs_pointing = []
file_needs_astrometry = []
if lights:
has_no_ra = np.array([True] * len(lights))
has_no_ha = np.array([True] * len(lights))
for ra_name in RA.names:
col_name = \
get_column_name_case_insensitive(ra_name, lights.colnames)
try:
has_no_ra &= (lights[col_name].mask)
except KeyError:
pass
for ha_name in feder.HA.names:
col_name = \
get_column_name_case_insensitive(ha_name, lights.colnames)
try:
has_no_ha &= lights[col_name].mask
except KeyError:
pass
file_needs_astrometry = list(lights['file'][lights['wcsaxes'].mask])
needs_minimal_pointing = has_no_ha | has_no_ra
file_needs_pointing = list(lights['file'][needs_minimal_pointing])
full_path = os.path.abspath(dir)
path_column = Column(data=[full_path] * len(file_info), name='Source path')
containing_dir = os.path.basename(full_path)
containing_dir_col = Column(data=[containing_dir] * len(file_info),
name='Source directory')
file_info.add_columns([path_column, containing_dir_col])
dir_info = {
'files': file_info,
'needs_filter': file_needs_filter,
'needs_pointing': file_needs_pointing,
'needs_object_name': file_needs_object_name,
'needs_astrometry': file_needs_astrometry
}
return dir_info
def lbcgo(raw_directory='./raw/',
image_directory='./',
lbc_chips=True,
lbcr=True,
lbcb=True,
filter_names=None,
bias_proc=False,
do_astrometry=True,
scamp_iterations=3,
verbose=True,
clean=True):
"""Process a directory of LBC data.
By default, all raw data are to be in the ./raw/ directory before processing (raw_directory='./raw/').
By default, all new images are written in the CWD (image_directory='./').
"""
# It will go something like this...
# Make sure the input directories have trailing slashes:
if raw_directory[-1] != '/':
raw_directory += '/'
if image_directory[-1] != '/':
image_directory += '/'
########## Collect basic image information
# Find the raw files we'll use
if lbcb & lbcr:
lbc_file_base = 'lbc?.*.*.fits*'
elif lbcb:
lbc_file_base = 'lbcb.*.*.fits*'
elif lbcr:
lbc_file_base = 'lbcr.*.*.fits*'
# What information do we want from the headers?
keywds = [
'object', 'filter', 'exptime', 'imagetyp', 'propid', 'lbcobnam',
'airmass', 'HA', 'objra', 'objdec'
]
##### Create an ImageFileCollection object to hold the raw data list.
if np.int(ccdproc.__version__[0]) == 2:
ic0 = ImageFileCollection(raw_directory,
keywords=keywds,
glob_include=lbc_file_base)
num_images = np.size(ic0.summary['object'])
# Exit if (for some reason) no images are found in the raw_directory.
if num_images == 0:
print('WARNING: No images found.')
return None
else:
raw_lbc_files = glob(raw_directory + lbc_file_base)
ic0 = ImageFileCollection(raw_directory,
keywords=keywds,
filenames=raw_lbc_files)
num_images = np.size(ic0.summary['object'])
# Exit if (for some reason) no images are found in the raw_directory.
if num_images == 0:
print('WARNING: No images found.')
return None
######### Create the master bias frame (if requested)
if bias_proc == True:
make_bias(ic0,
image_directory=image_directory,
raw_directory=raw_directory)
###### Define which chips to extract if default is chosen:
if lbc_chips == True:
lbc_chips = [1, 2, 3, 4]
###### Per filter:
#
# Step through the filters in the ImageCollection unless user gives filter
# list.
#
# For now nights with V-band filters in both LBCB and LBCR require running
# the code with 'lbcr=False' then 'lbcb=False' to avoid coadding the two
# images.
# TODO: Loop B/R to avoid issues w/V-band from LBCB/LBCR?
# TODO: Check for co-pointing files
if filter_names == None:
icX = ImageFileCollection(
raw_directory,
keywords=keywds,
filenames=(ic0.files_filtered(imagetyp='object')).tolist())
filter_names = icX.values('filter', unique=True)
# Loop through each of the filters
for filter in filter_names:
# Find the images with the filter of interest.
if verbose == True:
print('Processing {0} files.'.format(filter))
# List of images in the current filter
ic1 = ImageFileCollection(
raw_directory,
keywords=keywds,
filenames=(ic0.files_filtered(filter=filter)).tolist())
# Make master flat fields.
# Check to see if flat exists; if so, skip making the flat (time
# consuming)
flatname = 'flat.' + filter + '.fits'
if not os.path.lexists(flatname):
make_flatfield(ic1,
verbose=verbose,
raw_directory=raw_directory,
image_directory=image_directory)
else:
print('Using existing {0}'.format(flatname))
# Remove overscan, trim object files.
overfiles = go_overscan(ic1,
lbc_chips=lbc_chips,
verbose=verbose,
raw_directory=raw_directory,
image_directory=image_directory)
# Apply bias frames
if bias_proc == True:
# zero_files = go_bias(verbose=verbose)
print('')
# Image collection of object frames overscanned & bias subtracted
ic2 = ImageFileCollection(image_directory,
keywords=keywds,
filenames=overfiles)
# Apply the flatfields
flatfiles = go_flatfield(ic2,
return_files=True,
image_directory=image_directory,
verbose=verbose,
cosmiccorrect=False)
# Image collection of object frames overscanned & bias subtracted + flattened
ic3 = ImageFileCollection(image_directory,
keywords=keywds,
filenames=flatfiles)
# from IPython import embed ; embed()
# Create directories for extracting individual chips.
tgt_dirs, fltr_dirs = make_targetdirectories(
ic3, image_directory=image_directory, verbose=verbose)
# Extract individual chips
go_extractchips(fltr_dirs, lbc_chips, verbose=verbose)
# Register and coadd the images
if do_astrometry:
go_register(fltr_dirs,
lbc_chips=lbc_chips,
scamp_iterations=scamp_iterations)
# Let's do some clean-up.
if clean:
# We will eventually ...
#remove _over, _zero, _flat files.
print('')
else:
# We will eventually ...
# Move _over, _zero, _flat files.
print('')
def run_memory_profile(n_files, sampling_interval, size=None, sigma_clip=False,
combine_method=None, memory_limit=None):
"""
Try opening a bunch of files with a relatively low limit on the number
of open files.
Parameters
----------
n_files : int
Number of files to combine.
sampling_interval : float
Time, in seconds, between memory samples.
size : int, optional
Size of one side of the image (the image is always square).
sigma_clip : bool, optional
If true, sigma clip the data before combining.
combine_method : str, optional
Should be one of the combine methods accepted by
ccdproc.combine
memory_limit : int, optional
Cap on memory use during image combination.
"""
# Do a little input validation
if n_files <= 0:
raise ValueError("Argument 'n' must be a positive integer")
proc = psutil.Process()
print('Process ID is: ', proc.pid, flush=True)
ic = ImageFileCollection(str(TMPPATH))
files = ic.files_filtered(for_prof='yes', include_path=True)
kwargs = {'method': combine_method}
if sigma_clip:
kwargs.update(
{'sigma_clip': True,
'sigma_clip_low_thresh': 5,
'sigma_clip_high_thresh': 5,
'sigma_clip_func': np.ma.median,
'sigma_clip_dev_func': median_absolute_deviation}
)
ccd = CCDData.read(files[0])
expected_img_size = _calculate_size_of_image(ccd, None)
if memory_limit:
kwargs['mem_limit'] = memory_limit
pre_mem_use = memory_usage(-1, interval=sampling_interval, timeout=1)
baseline = np.mean(pre_mem_use)
print('Subtracting baseline memory before profile: {}'.format(baseline))
mem_use = memory_usage((combine, (files,), kwargs),
interval=sampling_interval, timeout=None)
mem_use = [m - baseline for m in mem_use]
return mem_use, expected_img_size