def makedark(files, output):
"""
Make dark image for NIRC2 data. Makes a calib/ directory
and stores all output there. All output and temporary files
will be created in a darks/ subdirectory.
files: integer list of the files. Does not require padded zeros.
output: output file name. Include the .fits extension.
"""
redDir = os.getcwd() + "/" # Reduce directory.
curDir = redDir + "calib/"
darkDir = util.trimdir(curDir + "darks/")
rawDir = util.trimdir(os.path.abspath(redDir + "../raw") + "/")
util.mkdir(curDir)
util.mkdir(darkDir)
_out = darkDir + output
_outlis = darkDir + "dark.lis"
util.rmall([_out, _outlis])
darks = [rawDir + "n" + str(i).zfill(4) + ".fits" for i in files]
f_on = open(_outlis, "w")
f_on.write("\n".join(darks) + "\n")
f_on.close()
ir.unlearn("imcombine")
ir.imcombine.combine = "median"
ir.imcombine.reject = "sigclip"
ir.imcombine.nlow = 1
ir.imcombine.nhigh = 1
ir.imcombine("@" + _outlis, _out)
def combine(fdict, data_string):
iraf.imcombine.combine = 'average'
iraf.imcombine.reject = 'avsigclip'
iraf.imcombine.lsigma = 3
name_dict = {}
if data_string[data_string.find('*')-1] == '_':
fill = ''
else:
fill = '_'
for f_ratio in fdict.keys():
name_dict[f_ratio] = []
for filt in fdict[f_ratio].keys():
name = data_string[:data_string.find('*')]+fill+filt+'F'+\
str(int(f_ratio*10))
for ftype in fdict[f_ratio][filt].keys():
iraf.imcombine(','.join(fdict[f_ratio][filt][ftype]),\
name+ftype[0]+'.fits')
name_dict[f_ratio].append(name+ftype[0]+'.fits')
return name_dict
def ImgCombineWithZeroFloating(imglistfname,outputfile,cmethod="median",czero="median",
creject="avsigclip",cstatsection='[150:900,150:900]'):
""" Returns the combined image with actuall average median flux,
It does zero scaling only for sigma rejection of stars. This is needed to remove faint
stars in rejection algorithm when the background sky itself is varying from frame to frame. """
iraf.imcombine.unlearn()
Xmin=float(cstatsection[1:-1].split(',')[0].split(':')[0]) #Everything now in fits coordinates
Xmax=float(cstatsection[1:-1].split(',')[0].split(':')[1])
Ymin=float(cstatsection[1:-1].split(',')[1].split(':')[0])
Ymax=float(cstatsection[1:-1].split(',')[1].split(':')[1])
if czero == "median" : statfunction = np.median
elif czero == "average" : statfunction = np.mean
else :
print('Error: czero should be median or average. Unknown option {0}'.format(czero))
raise
with open(imglistfname,'r') as imgfile:
statlist=[]
for img in imgfile:
img = img.rstrip()
statlist.append(statfunction(fits.getdata(img)[Ymin-1:Ymax,Xmin-1:Xmax]))
print('{0} of images: {1}'.format(czero,str(statlist)))
statAvg=np.mean(statlist)
Zeroshifts= statAvg - np.array(statlist)
print('Zeroshifts of images: {0} :: ImgAvg ={1}'.format(str(Zeroshifts),statAvg))
with open(outputfile+'_zeroshifts.txt','w') as zeroshiftFILE:
for shift in Zeroshifts:
zeroshiftFILE.write('{0} \n'.format(shift))
# Now call iraf imcombine with zero scaling
iraf.imcombine(input='@'+imglistfname, output=outputfile, combine=cmethod,
reject=creject, statsec=cstatsection, zero='@'+outputfile+'_zeroshifts.txt')
def align_combine(fitsdir, myfilter, examine=True):
from pyraf import iraf
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.apphot(_doprint=0)
os.chdir(fitsdir)
listfiles = glob.glob(myfilter)
listfiles.sort()
if (examine):
print "Opening ",listfiles[0]," to examine."
iraf.imexamine(input=listfiles[0], \
logfile="coords.dat", \
keeplog="yes")
with open("align.list",'w') as f:
for i in listfiles:
f.write(i+"\n")
print "Aligning with reference:",listfiles[0]
iraf.imalign( input = "@align.list", referenc= listfiles[0], coords = "coords.dat", output = "*****@*****.**")
listfiles = glob.glob("a_"+myfilter)
listfiles.sort()
with open("comb.list",'w') as f:
for i in listfiles:
f.write(i+"\n")
print "Combining"
iraf.imcombine(input = "@comb.list",\
output = "out.fits",\
combine= "median")
def combine_cube_frames(cubefile, range_pairs, target_dir):
dirname, filename = os.path.split(cubefile)
filebase = filename.rsplit('.', 1)[0]
if len(filebase) > 8:
warn("IRAF doesn't like long filenames. "
"Consider shortening the cube filename ({0})".format(filebase))
outfiles = []
for fromidx, toidx in range_pairs:
inlst = os.path.join(target_dir, 'in.lst')
infiles = []
for i in range(fromidx, toidx+1):
infiles.append(cubefile + "[*,*,{0}]".format(i))
f = open(inlst, 'w')
f.writelines(infiles)
f.write('\n')
f.close()
outfile = '{0}/{1}_{2}-{3}.fit'.format(target_dir, filebase, fromidx, toidx)
debug("imcombine input={input} output={output} combine=sum reject=none".format(
input="@{0}".format(inlst), #','.join(infiles),
output=outfile,
))
outfiles.append(outfile)
iraf.imcombine(
input=','.join(infiles),
output=outfile,
combine="sum",
reject="none",
# project='no', # IRAF wants bools specified / default is nonboolean?
# mclip='no',
)
return outfiles
def gen_dark(self,exptime):
iraf.imcombine.reject = 'none'
iraf.imcombine.combine = self.darkcombine
name = self.dark_dir+'/Combined_'+str(exptime)+'_DARK.fits'
iraf.imcombine(','.join(self.darks[exptime]['raw']),name)
self.darks[exptime]['combined'] = name
def cos_clear3(filenames):
outname = 'c3' + filenames[0]
if os.path.isfile(outname):
print outname, 'is already exist'
else:
inname = ''
for i in filenames:
inname = inname + ',' + i
inname = inname[1:]
print 'runing cos_clear3, imcombine...'
print 'make file', outname
iraf.imcombine(input = inname
, output = outname, headers = '', bpmasks = ''
, rejmasks = '', nrejmasks = '', expmasks = ''
, sigmas = '', imcmb = '$I', logfile = 'STDOUT'
, combine = 'average', reject = 'avsigclip', project = False
, outtype = 'real', outlimits = 'none', offsets = 'none'
, masktype = '', maskvalue = 0.0, blank = 0.0
, scale = 'none', zero = 'none', weight = 'exposure'
, statsec = '', expname = 'EXPTIME', lthreshold = 'INDEF'
, hthreshold = 'INDEF', nlow = 0, nhigh = 1
, nkeep = 1, mclip = True, lsigma = 3.0
, hsigma = 8.0, rdnoise = 'RDNOISE', gain = 'GAIN'
, snoise = 0.0, sigscale = 0.1, pclip = -0.5, grow = 0.0)
print 'display %s 1' % outname
iraf.display(image = outname, frame = 1)
valget = raw_input('Are you need to run crmedian?(y or n): ')
if valget == 'Y' or valget == 'y':
return cos_clear1([outname])
else:
return outname
def createFlatFiles():
import re
print "CreateFlatFiles start"
for f in FILTERS:
if(os.listdir(os.path.join(OUTPUTDIR, "flat", f))):
iraf.imcombine.setParam("input", os.path.join(OUTPUTDIR, "flat", f) + "/*.fits")
flatFile = os.path.join(OUTPUTDIR, "flat", f , "Flat.fits")
if os.path.exists(flatFile):
print("flatFile %s alreday exists deleting" % flatFile)
os.remove(flatFile)
iraf.imcombine.setParam("output", flatFile)
#from doc:
#http://www.iac.es/sieinvens/siepedia/pmwiki.php?n=HOWTOs.PythonianIRAF
#--> iraf.listpix(mode='ql') # confirms parameter
#--> iraf.listpix(mode='h') # doesn't ask for parameter...@@
iraf.imcombine(mode="h")
#NORMALIZE
#imstat
res = iraf.imstat(flatFile, Stdout=1)
print(res[0].strip())
print(res[1].strip())
resArray = re.split("\s+", res[1].strip())
#max value
#toDivValue = float(resArray[5])
#meanValue
toDivValue = float(resArray[2])
flatNormFile = os.path.join(OUTPUTDIR, "flat", f , "FlatNorm.fits")
if os.path.exists(flatNormFile):
print("flatNormFile %s alreday exists deleting" % flatNormFile)
os.remove(flatNormFile)
#divide by max value
iraf.imarith(flatFile, '/', toDivValue, flatNormFile)
else:
print("NO FLAT FILES for filter %s PRESENT" %f)
print "CreateFlatFiles end"
def cos_clear2(filenames):
outname = 'fake_' + filenames[0]
if os.path.isfile(outname):
print outname, 'is already exist'
else:
inname = filenames[0] + ',' + filenames[1]
print 'runing cos_clear2, imcombine...'
print 'make file', outname
iraf.imcombine(input = inname
, output = outname, headers = '', bpmasks = ''
, rejmasks = '', nrejmasks = '', expmasks = ''
, sigmas = '', imcmb = '$I', logfile = 'STDOUT'
, combine = 'average', reject = 'minmax', project = False
, outtype = 'real', outlimits = 'none', offsets = 'none'
, masktype = 'none', maskvalue = 0, blank = 0.0
, scale = 'exposure', zero = 'none', weight = 'none'
, statsec = '', expname = 'EXPTIME', lthreshold = 'INDEF'
, hthreshold = 'INDEF', nlow = 0, nhigh = 1
, nkeep = 1, mclip = True, lsigma = 3.0
, hsigma = 10.0, rdnoise = 'RDNOISE', gain = 'GAIN'
, snoise = 0.0, sigscale = 0.1, pclip = -0.5, grow = 0.0)
iraf.hedit(images = outname
, fields = 'ncombine', value = '0', add = True
, addonly = False, delete = False, verify = False
, show = True, update = True)
iraf.hedit(images = outname
, fields = 'EXPTIME', value = '0', add = True
, addonly = False, delete = False, verify = False
, show = True, update = True)
print 'display %s 1' % outname
iraf.display(image = outname, frame = 1)
filenames.append(outname)
return cos_clear3(filenames)
def makedark(files, output):
"""
Make dark image for NIRC2 data. Makes a calib/ directory
and stores all output there. All output and temporary files
will be created in a darks/ subdirectory.
files: integer list of the files. Does not require padded zeros.
output: output file name. Include the .fits extension.
"""
redDir = os.getcwd() + '/' # Reduce directory.
curDir = redDir + 'calib/'
darkDir = util.trimdir(curDir + 'darks/')
rawDir = util.trimdir(os.path.abspath(redDir + '../raw') + '/')
util.mkdir(curDir)
util.mkdir(darkDir)
_out = darkDir + output
_outlis = darkDir + 'dark.lis'
util.rmall([_out, _outlis])
darks = [rawDir + 'n' + str(i).zfill(4) + '.fits' for i in files]
f_on = open(_outlis, 'w')
f_on.write('\n'.join(darks) + '\n')
f_on.close()
ir.unlearn('imcombine')
ir.imcombine.combine = 'median'
ir.imcombine.reject = 'sigclip'
ir.imcombine.nlow = 1
ir.imcombine.nhigh = 1
ir.imcombine('@' + _outlis, _out)
def combine_subset(filter='G141', idx=np.array([0]), root='set1', use_scaled=True):
"""
Subset, get index array of objects to use from the "show_profile" function above
"""
from pyraf import iraf
bg_flt, bg_field, bg_val = np.loadtxt('background.%s.dat' %(filter), dtype=np.str, unpack=True)
weights = np.cast[float](bg_val)**2
fp = open('%s.%s.list' %(filter, root),'w')
fpw = open('%s.%s.weight' %(filter, root),'w')
for msk, wht in zip(bg_flt[idx], weights[idx]):
if os.path.exists(msk):
if use_scaled:
img = msk.replace('msk','msk.s')
else:
img = msk
fp.write('%s\n' %(img))
fpw.write('%.4f\n' %(wht))
#
fp.close()
fpw.close()
iraf.imcombine ( input = '@%s.%s.list' %(filter, root), output = 'combine.%s.%s' %(filter, root),
headers = '', bpmasks = '', rejmasks = '', nrejmasks = '',
expmasks = '', sigmas = '', logfile = 'STDOUT', combine = 'average',
reject = 'minmax', project = iraf.no, outtype = 'real',
outlimits = '', offsets = 'none', masktype = 'none',
maskvalue = '0', blank = 0.0, scale = 'none', zero = 'none',
weight = '@%s.%s.weight' %(filter, root), statsec = '', expname = '', lthreshold = 1e-04,
hthreshold = 100.0, nlow = 2, nhigh = 2, nkeep = 1,
mclip = iraf.yes, lsigma = 3.0, hsigma = 3.0, rdnoise = '0.',
gain = '1.', snoise = '0.', sigscale = 0.1, pclip = -0.5)
def imcombine(filelist,
out,
options,
bpmask=None,
reject="none",
nlow=None,
nhigh=None):
'''Convenience wrapper around IRAF task imcombine
Args:
filelist: The list of files to imcombine
out: The full path to the output file
options: Options dictionary
bpmask: The full path to the bad pixel mask
reject: none, minmax, sigclip, avsigclip, pclip
nlow,nhigh: Parameters for minmax rejection, see iraf docs
Returns:
None
Side effects:
Creates the imcombined file at location `out'
'''
#TODO: REMOVE Iraf and use python instead. STSCI Python has
# A builtin routine.
from pyraf import iraf
iraf.images()
filelist = [("%s[0]" % f) for f in filelist]
pars = iraf.imcombine.getParList()
iraf.imcombine.unlearn()
path = "flatcombine.lst"
f = open(path, "w")
for file in filelist:
f.write(file + "\n")
f.close()
s = ("%s," * len(filelist))[0:-1]
s = s % tuple(filelist)
f = open("flatcombinelog.txt", "w")
if reject == 'minmax':
t = iraf.imcombine("@%s" % path,
out,
Stdout=f,
reject=reject,
nlow=nlow,
nhigh=nhigh)
else:
t = iraf.imcombine(s, out, Stdin=filelist, Stdout=f, reject=reject)
f.close()
f = open("flatcombinelog.txt")
for line in f:
info(line.rstrip("\n"))
f.close()
iraf.imcombine.setParList(pars)
def make_dark_avg(images, out):
"""
Create an average dark frame from a set of images.
"""
filenames = [im.filename for im in images]
iraf.imcombine(input=",".join(filenames), output=out, Stdout=0,
combine="median")
return out
def generate_all_masterbias(target_dir, bias_dir_name):
""" Calculation of all the masterbias files.
This function search for bias files from current directory.
The bias images are located in specific directories that only
contains bias images and have a specific denomination, so searching
for bias files is searching these directories.
Once a directory for bias had been found a masterbias is calculated
with an average operation using all the bias files.
Args:
target_dir: Directory of the files.
bias_dir_name: Name of the directories that contain bias images.
"""
logging.info("Generating all masterbias files from %s ..." % target_dir)
# Walk from current directory.
for path, dirs, files in os.walk(target_dir):
# Check if current directory is for bias fits.
for dr in dirs:
if dr == bias_dir_name:
# Get the full path of the directory.
full_dir = os.path.join(path, dr)
logging.debug("Found a directory for 'bias': %s" % (full_dir))
# Get the list of files.
files = glob.glob(os.path.join(full_dir, WILDCARD_FIT_FILE))
logging.debug("Found %d bias files" % (len(files)))
# Build the masterbias file name.
masterbias_name = os.path.join(full_dir, MASTERBIAS_FILENAME)
# Check if masterbias already exists.
if os.path.exists(masterbias_name) == True:
logging.debug("Masterbias file exists '%s', so resume to next directory." %
(masterbias_name))
else:
# Put the files list in a string.
list_of_files = ",".join(files)
#show_bias_files_statistics(list_of_files)
# Combine all the bias files.
try:
logging.debug("Creating bias file: %s" %
masterbias_name)
iraf.imcombine(list_of_files, masterbias_name, Stdout=1)
except iraf.IrafError as exc:
logging.error("Error executing imcombine combining " + \
"bias with: %s" %
(list_of_files))
logging.error("Iraf error is: %s" % (exc))
def makeSkyFlat(dfView):
"""
dfView is a view from either the {J,H}rawFileList data frame
the view should contain dither positions of the same observation
this module will calculate the skyflat from the dither positions
and save it
"""
# gather up the images in a python list
images = dfView.file.values.tolist()
# they should all have the same date, just grab the first one
date = str(dfView.Date.values[0])
# organize the images and output file name in iraf-friendly ways
inputFiles = joinStrList(images)
outputSkyFlat = "scratch/" + date + "sky"
# use iraf imcombine to create the median skyflat
iraf.imcombine(
inputFiles,
outputSkyFlat,
headers="",
bpmasks="",
rejmasks="",
nrejmasks="",
expmasks="",
sigmas="",
imcmb="$I",
logfile="STDOUT",
combine="median",
reject="none",
project="no",
outtype="real",
outlimits="",
offsets="none",
masktype="none",
maskvalue="0",
blank=0.0,
scale="none",
zero="none",
weight="none",
statsec="",
expname="",
lthreshold="INDEF",
hthreshold="INDEF",
nlow=1,
nhigh=2,
nkeep=1,
mclip="yes",
lsigma=3.0,
hsigma=3.0,
rdnoise="0.",
gain="1.",
snoise="0.",
sigscale=0.1,
pclip=-0.5,
grow=0.0,
)
return
def make_dark_avg(images, out):
"""
Create an average dark frame from a set of images.
"""
filenames = [im.filename for im in images]
iraf.imcombine(input=",".join(filenames),
output=out,
Stdout=0,
combine="median")
return out
def combinelamp(lst):
iraf.noao()
iraf.imred()
iraf.ccdred()
iraf.imcombine(input='%ftb%ftb%@' + lst,
output='Lamp',
combine='sum',
reject='none')
print('<<<<<combine the lamp & generate the Lamp.fits>>>>>')
iraf.flpr()
def imcombine(group, output):
if type(group) == list:
group = (",".join(group)) # if group datatype is list
#else : print(group.split(','))
iraf.imcombine(group,
output=output,
combine=combinestr,
project="no",
reject="none",
scale="none",
zero="mode")
def imcombine(filelist, out, options, bpmask=None, reject="none", nlow=None,
nhigh=None):
'''Convenience wrapper around IRAF task imcombine
Args:
filelist: The list of files to imcombine
out: The full path to the output file
options: Options dictionary
bpmask: The full path to the bad pixel mask
reject: none, minmax, sigclip, avsigclip, pclip
nlow,nhigh: Parameters for minmax rejection, see iraf docs
Returns:
None
Side effects:
Creates the imcombined file at location `out'
'''
#TODO: REMOVE Iraf and use python instead. STSCI Python has
# A builtin routine.
from pyraf import iraf
iraf.images()
filelist = [("%s[0]" % f) for f in filelist]
pars = iraf.imcombine.getParList()
iraf.imcombine.unlearn()
path = "flatcombine.lst"
f = open(path, "w")
for file in filelist:
f.write(file + "\n")
f.close()
s = ("%s," * len(filelist))[0:-1]
s = s % tuple(filelist)
f = open("flatcombinelog.txt", "w")
if reject == 'minmax':
t = iraf.imcombine("@%s" % path, out, Stdout=f,
reject=reject, nlow=nlow, nhigh=nhigh)
else:
t = iraf.imcombine(s, out, Stdin=filelist, Stdout=f,
reject=reject)
f.close()
f=open("flatcombinelog.txt")
for line in f:
info(line.rstrip("\n"))
f.close()
iraf.imcombine.setParList(pars)
def reduceBias(bListFn, bMasterFn):
if not os.path.isfile(bMasterFn + ".fits"):
iraf.imcombine(
"@" + bListFn,
output=bMasterFn,
logfile=bMasterFn + ".log",
weight="none",
zero="none",
reject="crreject",
combine="median",
scale="none",
)
def createZeroFile():
print "CreateZeroFile start"
if(os.listdir(os.path.join(OUTPUTDIR, "bias"))):
iraf.imcombine.setParam("input", os.path.join(OUTPUTDIR, "bias") + "/*.fits")
zeroFile = os.path.join(OUTPUTDIR, "bias", "Zero.fits")
if os.path.exists(zeroFile):
os.remove(zeroFile)
iraf.imcombine.setParam("output", zeroFile)
iraf.imcombine()
else:
print("NO BIAS FILES PRESENT")
print "CreateZeroFile end"
def combineDithers(color):
"""grab the aligned images and sum them up into one"""
# grab the paths to the sky subtracted flattened aligned images
aligned = sorted(glob.glob("scratch/s-f-*_gregister.fits"))
# pick out the YYMMDD date from the file names
date = aligned[0][17:23]
# organize input and output in iraf-friendly ways
inputFiles = joinStrList(aligned)
outputFile = "reduced/" + date + "." + color + ".4U1543.s-f-a-c"
# use iraf imcombine to sum them all up
iraf.imcombine(
inputFiles,
outputFile,
headers="",
bpmasks="",
rejmasks="",
nrejmasks="",
expmasks="",
sigmas="",
imcmb="$I",
logfile="STDOUT",
combine="sum",
reject="none",
project="no",
outtype="real",
outlimits="",
offsets="none",
masktype="none",
maskvalue="0",
blank=0.0,
scale="none",
zero="none",
weight="none",
statsec="",
expname="",
lthreshold="INDEF",
hthreshold="INDEF",
nlow=1,
nhigh=2,
nkeep=1,
mclip="yes",
lsigma=3.0,
hsigma=3.0,
rdnoise="0.",
gain="1.",
snoise="0.",
sigscale=0.1,
pclip=-0.5,
grow=0.0,
)
return
def combinedarks():
darktimes = [10, 120, 20, 30, 300, 360, 4, 45, 5, 60, 90]
for t in darktimes:
prefix = "qdark_" + str(t) + "????.fits"
out = "dk" + str(t)
iraf.imcombine(prefix,
output=out,
combine="ave",
reject="ccdclip",
rdnoise=READNOISE,
gain=GAIN,
lsigma=3,
hsigma=3)
def createFlatFiles():
print "CreateFlatFiles start"
for f in FILTERS:
if(os.listdir(os.path.join(OUTPUTDIR, "flat", f))):
iraf.imcombine.setParam("input", os.path.join(OUTPUTDIR, "flat", f) + "/*.fits")
flatFile = os.path.join(OUTPUTDIR, "flat", f , "Flat.fits")
if os.path.exists(flatFile):
os.remove(flatFile)
iraf.imcombine.setParam("output", flatFile)
iraf.imcombine()
else:
print("NO FLAT FILES for filter %s PRESENT" %f)
print "CreateFlatFiles end"
def avg_images(images, out):
"""
Create an average image from a list of images using imcombine.
Writes to the file out.
"""
filelist = "tmp.txt"
f = open(filelist, "w")
for image in images:
f.write("%s\n" % image.filename)
f.close()
iraf.imcombine(input="@%s" % filelist, output=out, Stdout=1)
os.remove(filelist)
return out
def createZeroFile():
print "CreateZeroFile start"
if (os.listdir(os.path.join(OUTPUTDIR, "bias"))):
iraf.imcombine.setParam("input",
os.path.join(OUTPUTDIR, "bias") + "/*.fits")
zeroFile = os.path.join(OUTPUTDIR, "bias", "Zero.fits")
if os.path.exists(zeroFile):
os.remove(zeroFile)
iraf.imcombine.setParam("output", zeroFile)
iraf.imcombine()
else:
print("NO BIAS FILES PRESENT")
print "CreateZeroFile end"
def combinePyraf(input, output, median, scale):
"""
Combines given images using IRAF's imcombine.
"""
from pyraf import iraf
from pyraf.iraf import imred
reject = "sigclip"
combine = ""
if median: combine = "median"
else: combine = "average"
iraf.imcombine(input=input, output=output, combine=combine, reject=reject, scale=scale)
def imcombine(filelist, out, listfile=None, bpmask=None, reject="none",
nlow=None, nhigh=None):
"""Convenience wrapper around IRAF task imcombine
Args:
filelist (list of str): The list of files to imcombine
out (str): The full path to the output file
bpmask (str): The full path to the bad pixel mask
reject (str): none, minmax, sigclip, avsigclip, pclip
nlow,nhigh (int,int): Parameters for minmax rejection, see iraf docs
Returns:
None
Side effects:
Creates the imcombined file at location `out`
"""
#TODO: REMOVE Iraf and use python instead. STSCI Python has
# A builtin routine.
from pyraf import iraf
iraf.images()
filelist = [("%s[0]" % f) for f in filelist]
pars = iraf.imcombine.getParList()
iraf.imcombine.unlearn()
if listfile is None:
path = "flatcombine.lst"
else: path = listfile
f = open(path, "w")
for file in filelist:
f.write(file + "\n")
f.close()
s = ("%s," * len(filelist))[0:-1]
s = s % tuple(filelist)
if reject == 'minmax':
t = iraf.imcombine("@%s" % path, out, combine="average",
reject=reject, nlow=nlow, nhigh=nhigh)
elif reject == 'sigclip':
t = iraf.imcombine("@%s" % path, out, combine="average",
reject=reject, lsigma=nlow, hsigma=nhigh)
else:
t = iraf.imcombine(s, out, Stdin=filelist, Stdout=1, combine="average",
reject=reject)
iraf.imcombine.setParList(pars)
def main(raw_dir):
if not os.path.exists("{}/step1.tmp".format(os.getcwd())):
print "Setting up directory"
list_dict = setup_dir(raw_dir)
print "OT'ing"
iraf.ccdproc(
"@all_raw.lst",
output="@all_ot.lst",
overscan=True,
trim=True,
zerocor=False,
darkcor=False,
flatcor=False,
illumco=False,
biassec="image",
trimsec="image",
zero="",
interact=True,
order=100,
)
cull_bias(list_dict["zeros"], "zeros.lst")
os.system("touch {}/step1.tmp".format(os.getcwd()))
return
print "Making master bias"
iraf.zerocombine("@zeros.lst", output="Zero.fits", combine="average", reject="crreject")
print "Subtracting bias"
iraf.ccdproc("@all_otz.lst", overscan=False, trim=False, zerocor=True, zero="Zero.fits")
detect_flats(os.getcwd())
flats_to_make = glob("dflat*.lst")
for flat in flats_to_make:
name = "dFlat_{}.fits".format(flat.split(".lst")[0].split("_")[1])
print "Making flat {}".format(name)
iraf.flatcombine("@{}".format(flat), output=name, combine="average", reject="crreject")
try:
print "Making master dark"
iraf.darkcombine("@darks.lst", output="Dark.fits", combine="average", reject="crreject")
except Exception as e:
print "\t{}".format(e)
print "\tNO DARK FOUND"
print "Making master comp"
iraf.imcombine("@comps.lst", "Comp.fits", combine="average", reject="crreject")
return
def combine_images(args):
""" Combine images using the wcs in their headers to match the pixels.
:param args:
:return:
"""
utilities.if_exists_remove(args.output)
input_names = ",".join(args.input)
# Unfortunately the mean is considered the only average in IRAF... (sic!)
if args.average == "mean":
args.average = "average"
iraf.imcombine(input_names, output=args.output, scale=args.scale.lower(), combine=args.average.lower(),
offsets="wcs")
def finalMergeCubes(mergeType, over):
"""
Merge final merged cubes from all observations.
"""
# Load data from the previous step.
path = os.getcwd()
with open('mergedInfo.txt') as data_file:
mergedData = json.load(data_file)
mergedCubes = mergedData['mergedCubes']
Merged = mergedData['Merged']
if len(mergedCubes)>1:
os.chdir(Merged)
iraffunctions.chdir(Merged)
gratlist = []
for i in range(len(mergedCubes)):
cubeheader = astropy.io.fits.open(mergedCubes[i])
grat = cubeheader[0].header['GRATING']
gratlist.append(grat)
print "gratlist is: ", gratlist
# TODO(nat): right now we do more final merges here than we have to. Eg, if there are three H
# grating directories in gratlist, we will do a final merge three times. We should only be doing this once!
# Right now it is only a problem when overwrite is turned on but it should still be fixed.
for n in range(len(gratlist)): # For each unique grating
# Grab the indices of the cubes associated with that grating.
indices = [k for k, x in enumerate(gratlist) if x==gratlist[n]]
newcubelist = []
for ind in indices:
newcubelist.append(mergedCubes[ind])
print newcubelist
# Do some housekeeping before the final cube merging.
resizeAndCenterCubes(newcubelist, over)
makeWavelengthOffsets(newcubelist, grat)
for i in range(len(newcubelist)):
# Build an input string containing all the cubes to combine.
if i==0:
inputstring = newcubelist[i]+'[1]'
else:
inputstring += ','+newcubelist[i]+'[1]'
if os.path.exists('temp_merged'+gratlist[n][0]+'.fits'):
if over:
iraf.delete('temp_merged'+gratlist[n][0]+'.fits')
iraf.imcombine(inputstring, output = 'temp_merged'+gratlist[n][0]+'.fits', combine = mergeType, offsets = 'waveoffsets'+grat[0]+'.txt')
iraf.fxcopy(input=newcubelist[0]+'[0], temp_merged'+gratlist[n][0]+'.fits', output = 'TOTAL_merged'+gratlist[0][0]+'.fits')
else:
logging.info('Output exists and -over- not set - skipping final cube merge')
else:
iraf.imcombine(inputstring, output = 'temp_merged'+gratlist[n][0]+'.fits', combine = mergeType, offsets = 'waveoffsets'+grat[0]+'.txt')
iraf.fxcopy(input=newcubelist[0]+'[0], temp_merged'+gratlist[n][0]+'.fits', output = 'TOTAL_merged'+gratlist[n][0]+'.fits')
os.chdir(path)
def gen_direct(f_ratio,filt):
iraf.imcombine.combine = self.datacombine
iraf.imcombine.reject = self.datareject
iraf.imcombine.lsigma = self.rejectpar
self.sub_darks(self.direct[f_ratio][filt]['raw'])
name = self.direct_dir+\
'/Combined_'+filt+'F'+str(int(focal_ratio*10))+'d.fits'
if len(self.direct[f_ratio][filt]['raw'].keys()) > 1:
print 'WARNING, DIRECT IMAGES MAY HAVE MULTIPLE EXPS PER FILT'
exp = self.direct[f_ratio][filt]['raw'].keys()[0]
iraf.imcombine(','.join(self.direct[f_ratio][filt]['raw'][exp]['ds']),\
name)
self.direct[f_ratio][filt]['combined'] = name
def CombineImages(imglist,output,method='median',zero='none' ,scale='none',norm=False,
reject="avsigclip", statsec='[150:900,150:900]'):
""" Combined the input list of images and write to output fits file. """
iraf.imcombine.unlearn()
imglistfname = os.path.splitext(output)[0]+'.comblist'
with open(imglistfname,'w') as imgs2combinefile:
imgs2combinefile.write('\n'.join(imglist)+'\n')
# Now call iraf imcombine with zero scaling
combineoutputfile = os.path.splitext(output)[0]+'_un.fits' if norm else output
iraf.imcombine(input='@'+imglistfname, output=combineoutputfile, combine=method,
reject=reject, statsec=statsec, scale=scale, zero=zero)
if norm:
mediancounts = np.median(fits.getdata(combineoutputfile))
iraf.imarith(operand1=combineoutputfile,op='/',operand2=mediancounts,result=output)
def viewWhiteIm(self, inIm, **kwargs):
print "\nDISPLAYING WHITE-LIGHT IMAGE"
idx = inIm.find(".")
#whiteIm = inIm[:idx] + "_2d.fits"
whiteIm = "whiteIm_" + inIm
iraf.imdelete(whiteIm)
iraf.imcombine(inIm + "[SCI]", whiteIm, project="yes", combine="sum", \
logfile="")
iraf.display(whiteIm)
return
def createFlatFiles():
print "CreateFlatFiles start"
for f in FILTERS:
if (os.listdir(os.path.join(OUTPUTDIR, "flat", f))):
iraf.imcombine.setParam(
"input",
os.path.join(OUTPUTDIR, "flat", f) + "/*.fits")
flatFile = os.path.join(OUTPUTDIR, "flat", f, "Flat.fits")
if os.path.exists(flatFile):
os.remove(flatFile)
iraf.imcombine.setParam("output", flatFile)
iraf.imcombine()
else:
print("NO FLAT FILES for filter %s PRESENT" % f)
print "CreateFlatFiles end"
def imcombine(imlist_name,
newimage,
combine='median',
reject='sigclip',
scale='none',
zero='mode'):
from pyraf import iraf
import os, sys
import glob
image_to_com = glob.glob(imlist_name)
iraf.imcombine(','.join(image_to_com),
newimage,
combine=combine,
reject=reject,
scale=scale,
zero=zero)
def combimage(images, bandpass, ditgroupcounter, target):
imagesinirafformat = str(images).replace("', '", ",").replace("['",
"").replace(
"']", "")
finalimage = images[0][0:17] + target + '.' + bandpass + '.' + images[
0].split('/')[-1][8:14] + '.' + str(ditgroupcounter) + '.fits'
iraf.imcombine.unlearn()
iraf.imcombine(imagesinirafformat,
finalimage,
combine='median',
mclip='yes',
reject='none',
lsigma=3,
hsigma=3,
nlow=1,
nhigh=1,
nkeep=1)
def imcombine_flatten(cube, sci_ext, var_ext):
"""
Imcombine both variance and science extensions to flatten the cube to 2D
INPUT: CUBE (scrop_proc/IC225_3D_{}_{}.fits)
OUTPUT: SCI_EXT (scrop_proc/IC225_2D_sci_{}_{}.fits), VAR_EXT (scrop_proc/IC225_2D_var_{}_{}.fits)
"""
if os.path.exists(sci_ext):
print('File {} already exists'.format(sci_ext))
return
if os.path.exists(var_ext):
print('File {} already exists'.format(var_ext))
return
from pyraf import iraf
iraf.imcombine('{}[sci]'.format(cube), sci_ext, project="yes")
iraf.imcombine('{}[var]'.format(cube), var_ext, project="yes")
def flat_combine(infiles_comma, overwrite=False):
print('\n#############################')
print('Combining the flat frames.')
infiles = infiles_comma.split(',')
combinedname = fits.getval(infiles[0], 'FRAMEID') + '.fcmb.fits'
if os.path.exists(combinedname):
if overwrite:
try:
os.remove(combinedname)
except:
pass
else:
print('\t Combined flat frame already exists. ' + combinedname)
print('\t This precedure is skipped.')
return combinedname
iraf.imcombine(infiles_comma, combinedname, combine='median', reject='no')
return combinedname
def generate_masterflat(path, flat_files, masterflat_name, masterbias_name):
"""Generates a master flat from the flat files_to_flat received.
Args:
path: Full source path of the flat files_to_flat.
files_to_flat: List of flat files_to_flat.
masterflat_name: The name of the masterflat file.
masterbias_name: The name of the masterbias file.
"""
logging.debug("Creating masterflat: %s" % (masterflat_name))
# Check that there is not any previous temporary file in the path,
# it could exists if a previous execution was terminated just before
# removing them.
remove_temporary_files(path)
# Get the fit files in the directory, those should be only the flat ones.
flat_files = glob.glob(os.path.join(path, "*." + FIT_FILE_EXT))
reduce_flats(flat_files, masterbias_name)
# Normalize the flats passing the list of flat files,
normalize_flats(flat_files)
norm_files = [
os.path.join(path, f) for f in os.listdir(path)
if f.endswith(NORM_FILE_SUFFIX)
]
# Put the normalized flat files list in a string to be used with
# imcombine.
string_of_norm_files = ",".join(norm_files)
try:
iraf.imcombine(string_of_norm_files, masterflat_name, Stdout=1)
except iraf.IrafError as exc:
logging.error("Error executing imcombine combining flats with: %s" %
(string_of_norm_files))
finally:
remove_temporary_files(path)
def generate_masterflat(path, flat_files, masterflat_name, masterbias_name):
"""Generates a master flat from the flat files_to_flat received.
Args:
path: Full source path of the flat files_to_flat.
files_to_flat: List of flat files_to_flat.
masterflat_name: The name of the masterflat file.
masterbias_name: The name of the masterbias file.
"""
logging.debug("Creating masterflat: %s" % (masterflat_name))
# Check that there is not any previous temporary file in the path,
# it could exists if a previous execution was terminated just before
# removing them.
remove_temporary_files(path)
# Get the fit files in the directory, those should be only the flat ones.
flat_files = glob.glob(os.path.join(path, "*." + FIT_FILE_EXT))
reduce_flats(flat_files, masterbias_name)
# Normalize the flats passing the list of flat files,
normalize_flats(flat_files)
norm_files = [os.path.join(path,f) for f in os.listdir(path) if f.endswith(NORM_FILE_SUFFIX)]
# Put the normalized flat files list in a string to be used with
# imcombine.
string_of_norm_files = ",".join(norm_files)
try:
iraf.imcombine(string_of_norm_files, masterflat_name, Stdout=1)
except iraf.IrafError as exc:
logging.error("Error executing imcombine combining flats with: %s" %
(string_of_norm_files))
finally:
remove_temporary_files(path)
def combine(self):
iraf.imcombine.combine = self.datacombine
iraf.imcombine.reject = self.datareject
iraf.imcombine.lsigma = self.rejectpar
for fiber_pos in self.ratios.keys():
for filt in self.ratios[fiber_pos]['data'].keys():
for ftype in self.ratios[fiber_pos]['data'][filt].keys():
name = self.ratios[fiber_pos]['data'][filt][ftype]['raw'][0]
name = name[:name.rfind('.0')]+'.fits'
mintime = min(self.ratios[fiber_pos]['data'][filt][ftype]['obstimes'])
maxtime = max(self.ratios[fiber_pos]['data'][filt][ftype]['obstimes'])
maxtime += float(self.ratios[fiber_pos]['data'][filt][ftype]['exptime'])
iraf.imcombine(\
','.join(self.ratios\
[fiber_pos]['data'][filt][ftype]['raw']),\
name)
self.ratios[fiber_pos]['data'][filt][ftype]['final'] = name
iraf.nhedit(name,'STARTIME',mintime,'Start time of first combined image',addonly=True)
iraf.nhedit(name,'ENDTIME',maxtime,'End time of last combined image',addonly=True)
def combineflats(inflats, outflat, outflatdc, darkflat, flat_sigmas=None):
ir.imcombine("@" + inflats,
output=outflat,
combine="average",
reject="crreject",
scale="median",
weight="median",
bpmasks="") # sigmas=flat_sigmas
ns.write_exptime(outflat, itime=itime)
print(outflat)
ir.ccdproc(outflat,
output=outflatdc,
ccdtype="",
fixpix="no",
overscan="no",
trim="no",
zerocor="no",
darkcor="yes",
flatcor="no",
dark=darkflat)
def cmd_imageCube(self, the_command):
'''This function can be used to pull a series of images from the camera and coadd them in a simple way.
This is slightly better process for measuring the position of a star for the purposes of guiding.
In essence, this will take 10 images, average them and create a master image for analysis to be perfomed on.'''
comands = str.split(the_command)
if len(comands) != 3:
return 'Please specify the name of the final image and the number of images to median through. Alternatively, specify "high" instead of the number of images to acquire a high enough number of average over scintilation.'
if comands[2] == 'high': nims = 30 #3E4/self.set_values[2]
else:
try:
nims = int(comands[2])
except Exception:
return 'Unable to convert number of images to integer'
#make upperlimit images and average combine them.
upperlimit = int(nims)
base_filename = comands[1]
if base_filename in commands.getoutput('ls program_images/'):
os.system('rm program_images/' + base_filename + '*')
print 'Starting to capture images'
capture = self.capture_images('program_images/' + base_filename,
upperlimit,
show=False)
if not capture: return 'ERROR capturing images'
print 'Finished capturing images'
self.check_if_file_exists('program_images/' + base_filename + '.fits')
self.check_if_file_exists('program_images/inlist')
iraf.images(_doprint=0)
os.system('ls program_images/' + base_filename + '_*.fits > inlist')
try:
iraf.imcombine(input='@inlist',
output='program_images/' + base_filename + '.fits',
combine='average',
reject='none',
outtype='integer',
scale='none',
zero='none',
weight='none')
except Exception:
return 'Could not combine images'
return 'Final image created. It is image program_images/' + base_filename + '.fits'
def MakecBias(bias_dir, outfile): ########################## # Combine bias frames ########################## bframes = filter(fits_filter, os.listdir(bias_dir)) bframes = [os.path.join(bias_dir,i) for i in bframes] tfile = tempfile.mkstemp() os.write(tfile[0],"\n".join(bframes)+"\n") os.close(tfile[0]) iraf.imcombine.unlearn() # Default parameter list iraf.imcombine.input = "@" + tfile[1] # comma delimited list of input files iraf.imcombine.output = outfile # output file iraf.imcombine.combine = "median" # Median combine bias frames iraf.imcombine.mode = "h" # Run imcombine iraf.imcombine() os.remove(tfile[1])
def mygeotran(input,
output,
database,
transforms,
geometry='geometric',
interpolant="linear"):
# First, we get the shifts from the database
f = open(database, 'r')
for line in f.readlines():
if re.search('xshift', line):
xshift = float(string.split(line)[1])
elif re.search('yshift', line):
yshift = float(string.split(line)[1])
f.close()
f = open('offsets.dat', 'w')
print >> f, 0, 0
print >> f, xshift + 1, yshift + 1
f.close()
# Now we make a blank holder for the mapped image
nuke('placeholder.fits')
iraf.imcombine(input + ',' + input,
'placeholder.fits',
offsets='offsets.dat')
iraf.imarith('placeholder.fits', '*', 0.0, 'placeholder.fits')
xsize, ysize = get_image_size(input)
# next, copy the input into this place holder
iraf.imcopy(input + '[*,*]',
'placeholder.fits[1:%d,1:%d]' % (xsize, ysize))
# now run geogran on this padded image
iraf.geotran(input='placeholder.fits',
output=output,
database=database,
transforms=transforms,
geometry=geometry,
interpolant=interpolant)
def gen_darks(dark_dict, needs, dark_dir):
dark_list = glob.glob(dark_dir+'/DARK*.FIT')
dark_dict = {}
for dark in dark_list:
exptime = pyfits.open(dark)[exten].header['EXPTIME']
if exptime not in dark_dict.keys():
dark_dict[exptime] = []
dark_dict[exptime].append(dark)
iraf.imcombine.combine = 'median'
iraf.imcombine.reject = 'none'
for exp in needs:
iraf.imcombine(','.join(dark_dict[exp]),\
dark_dir+'/Combined_'+str(exp)+'_DARK.fits')
dark_dict[exp] = dark_dir+'/Combined_'+str(exp)+'_DARK.fits'
return dark_dict
def makesky(images, withmask):
imagesinirafformat = str(images).replace("', '", ",").replace("['",
"").replace(
"']", "")
if withmask == 0:
skyname = 'sky0.' + images[0][
6:12] + '.fits' # so the format will be sky0.yymmdd.fits
iraf.imcombine.unlearn()
iraf.imcombine(imagesinirafformat,
skyname,
combine='median',
reject='minmax',
nlow=0,
nhigh=1) #, scale=scales)
elif withmask == 1:
skyname = 'sky1.' + images[0][
7:13] + '.fits' # so the format will be sky1.yymmdd.fits
iraf.imcombine.unlearn()
iraf.imcombine(
imagesinirafformat, skyname, combine='median',
masktype='!OBJMASK') #, reject='minmax', nlow=0, nhigh=1)
return skyname
def createFlatFiles():
import re
print "CreateFlatFiles start"
for f in FILTERS:
if (os.listdir(os.path.join(OUTPUTDIR, "flat", f))):
iraf.imcombine.setParam(
"input",
os.path.join(OUTPUTDIR, "flat", f) + "/*.fits")
flatFile = os.path.join(OUTPUTDIR, "flat", f, "Flat.fits")
if os.path.exists(flatFile):
print("flatFile %s alreday exists deleting" % flatFile)
os.remove(flatFile)
iraf.imcombine.setParam("output", flatFile)
#from doc:
#http://www.iac.es/sieinvens/siepedia/pmwiki.php?n=HOWTOs.PythonianIRAF
#--> iraf.listpix(mode='ql') # confirms parameter
#--> iraf.listpix(mode='h') # doesn't ask for parameter...@@
iraf.imcombine(mode="h")
#NORMALIZE
#imstat
res = iraf.imstat(flatFile, Stdout=1)
print(res[0].strip())
print(res[1].strip())
resArray = re.split("\s+", res[1].strip())
#max value
#toDivValue = float(resArray[5])
#meanValue
toDivValue = float(resArray[2])
flatNormFile = os.path.join(OUTPUTDIR, "flat", f, "FlatNorm.fits")
if os.path.exists(flatNormFile):
print("flatNormFile %s alreday exists deleting" % flatNormFile)
os.remove(flatNormFile)
#divide by max value
iraf.imarith(flatFile, '/', toDivValue, flatNormFile)
else:
print("NO FLAT FILES for filter %s PRESENT" % f)
print "CreateFlatFiles end"
