def extract_spec2():
infile = workdir + '/maps/standards/HD221246_K3III.fits'
specInfo, hdr = pyfits.getdata(infile, header=True)
wave = specInfo[0,:] * 1e3 # in nm
spec = specInfo[1,:]
print wave[0:10]
print wave[-10:]
print spec
crpix1 = 1
crval1 = wave[0]
cdelt1 = wave[1] - wave[0]
cunit1 = 'nm'
tmp = np.arange(len(spec), dtype=float)
tmp = tmp*cdelt1 + crval1
print tmp[0:10]
print tmp[-10:]
hdr.update('CRPIX1', crpix1)
hdr.update('CRVAL1', crval1)
hdr.update('CDELT1', cdelt1)
hdr.update('CUNIT1', cunit1)
fitsFile = workdir + 'maps/test_spec_standard.fits'
ir.imdelete(fitsFile)
pyfits.writeto(fitsFile, spec, header=hdr)
def calibFlux(self, inIm, **kwargs):
print "\nCALIBRATING FLUX IN " + inIm
# determine which extinction file to use
hdu = fits.open(inIm)
if hdu[0].header["OBSERVAT"] == "Gemini-North":
self.extDat = "gmos$calib/mkoextinct.dat"
else:
self.extDat = "onedstds$ctioextinct.dat"
# determine which sensitivity function to use
centWave = str(int(hdu[0].header["CENTWAVE"]))
sFunc = self.sFunc.replace("X", centWave)
hdu.close()
# delete previous flux calibrated spectra and generate new one
iraf.imdelete("c" + inIm)
iraf.gscalibrate(inIm,
sfunction=sFunc,
extinction=self.extDat,
**kwargs)
# view the result
print "Displaying calibrated data cube"
self.viewCube("c" + inIm, version="1")
return
def extract_spec2():
infile = workdir + '/maps/standards/HD221246_K3III.fits'
specInfo, hdr = pyfits.getdata(infile, header=True)
wave = specInfo[0,:] * 1e3 # in nm
spec = specInfo[1,:]
print wave[0:10]
print wave[-10:]
print spec
crpix1 = 1
crval1 = wave[0]
cdelt1 = wave[1] - wave[0]
cunit1 = 'nm'
tmp = np.arange(len(spec), dtype=float)
tmp = tmp*cdelt1 + crval1
print tmp[0:10]
print tmp[-10:]
hdr.update('CRPIX1', crpix1)
hdr.update('CRVAL1', crval1)
hdr.update('CDELT1', cdelt1)
hdr.update('CUNIT1', cunit1)
fitsFile = workdir + 'maps/test_spec_standard.fits'
ir.imdelete(fitsFile)
pyfits.writeto(fitsFile, spec, header=hdr)
def subScatLgt(self, inIm, mask, **kwargs):
print "\nSUBTRACTING SCATTERED LIGHT FROM", inIm
pref = kwargs["prefix"]
# allow user to iterate over orders used to model scattered light
while True:
fixScat = bool(
input(
"\nDoes subtraction of scattered light need improvement? (True/False): "
))
if fixScat:
# request orders from user
xOrders = raw_input("X orders (csv): ")
yOrders = raw_input("Y orders (csv): ")
# model and subtract the light, and view the result
iraf.imdelete(pref + inIm)
iraf.gfscatsub(inIm, mask, xorder=xOrders, yorder=yOrders, \
**kwargs)
utils.examIm(pref + inIm + "[SCI]", frame=1)
else:
break
return
def viewCubeMov(self, inCube, step=10, **kwargs):
print "\nSTEPPING THROUGH " + inCube
# obtain number of wavelength samples in cube
hdul = fits.open(inCube)
nWave = hdul["SCI"].data.shape[0]
# step through cube, summing images within each chunk
cubeChunk = "chunk.fits"
for i in range(0, nWave, step):
iraf.imdelete(cubeChunk)
if i + 9 > nWave - 1:
lastFrame = str(nWave - 1)
else:
lastFrame = str(i + 9)
iraf.imcombine(inCube + "[SCI][*,*," + str(i) + ":" + lastFrame + \
"]", cubeChunk, project="yes", combine="sum", logfile="")
print "Displaying chunk [" + str(i) + ":" + lastFrame + "]"
iraf.display(cubeChunk, frame="1", contrast=0.)
time.sleep(2)
# delete the last wavelength chunk and close the cube
iraf.imdelete(cubeChunk)
hdul.close()
return
def getMasks(filename):
#Delete the old masks, if any exist.
iraf.imdelete(filename+'_Handmask.fits')
iraf.imdelete(filename+'_HaHandmask.fits')
#Extract the mask from the masked R image, using a handmasked image if one exists otherwise using the masked image
maskValue = 0.0
try:
maskedRImage = fits.getdata(filename+'_Handmasked.fits')
except IOError:
maskedRImage = fits.getdata(filename+'Masked.fits')
manyZeros = np.zeros_like(maskedRImage)
manyOnes = np.ones_like(maskedRImage)
print 'Writing R mask'
RmaskPixels = np.where((maskedRImage!=maskValue),manyZeros,manyOnes)
hdu=fits.PrimaryHDU(RmaskPixels)
hdulist = fits.HDUList([hdu])
hdulist.writeto(filename+'_Handmask.fits')
#Do the same for the Ha image
print 'Writing Ha mask'
try:
maskedHaImage = fits.getdata(filename+'_HaHandmasked.fits')
except IOError:
maskedHaImage = fits.getdata(filename+'_Ha.fits')
HaMaskPixels = np.where((maskedHaImage!=maskValue),manyZeros,manyOnes)
hdu=fits.PrimaryHDU(HaMaskPixels)
hdulist = fits.HDUList([hdu])
hdulist.writeto(filename+'_HaHandmask.fits')
def quartz_divide(science_list,object_match):
'''Divides science frames by user-selected quartz frames'''
for obj in science_list:
if len(object_match[obj]) > 1:
qtzinpt = ''
for qtz in object_match[obj]:
qtzinpt += qtz +','
iraf.imcombine(input=qtzinpt,output='tempquartz')
iraf.imarith(operand1=obj,operand2='tempquartz',op='/',result='f'+obj)
heditstr = 'Flat field images are '+qtzinpt[:-1]
iraf.imdelete(images='tempquartz',go_ahead='yes',verify='no')
if len(heditstr) > 65:
nfields = int(len(heditstr)/65) #Declare int for py3 compatibility
for ii in range(nfields+1):
writestr = heditstr[(ii*65):(ii+1)*65]
iraf.hedit(images='f'+obj,fields='flatcor'+str(ii),value=writestr,add='yes',verify='No')
else:
iraf.hedit(images='f'+obj,fields='flatcor',value=heditstr,add='yes',verify='No')
else:
iraf.imarith(operand1=obj,operand2=object_match[obj][0],op='/',result='f'+obj)
heditstr = 'Flat field image is '+object_match[obj][0]
if len(heditstr) > 65:
nfields = int(len(heditstr)/65) #Declare int for py3 compatibility
for ii in range(nfields+1):
writestr = heditstr[(ii*65):(ii+1)*65]
iraf.hedit(images='f'+obj,fields='flatcor'+str(ii),value=writestr,add='yes',verify='No')
else:
iraf.hedit(images='f'+obj,fields='flatcor',value=heditstr,add='yes',verify='No')
return
def extractSpec(self, inIm, **kwargs):
# remove previous spectra and extract new ones
iraf.imdelete("e" + inIm, verify="yes")
iraf.gfextract(inIm, **kwargs)
return
def cleanCosRays(self, inIm, pref="x", **kwargs):
print "\nCLEANING COSMIC RAYS FROM " + inIm
iraf.imdelete(pref + inIm)
iraf.gemcrspec(inIm, pref + inIm, **kwargs)
return
def extract_spec():
cubefits = pyfits.open(datadir + cuberoot + '.fits')
cube = cubefits[0].data
cubehdr = cubefits[0].header
spec = cube[11, 47, :]
fitsFile = workdir + 'maps/test_spec_11_47.fits'
ir.imdelete(fitsFile)
pyfits.writeto(fitsFile, spec, header=cubehdr)
def extract_spec():
cubefits = pyfits.open(datadir + cuberoot + '.fits')
cube = cubefits[0].data
cubehdr = cubefits[0].header
spec = cube[11, 47, :]
fitsFile = workdir + 'maps/test_spec_11_47.fits'
ir.imdelete(fitsFile)
pyfits.writeto(fitsFile, spec, header=cubehdr)
def correctQE(self, inIm, **kwargs):
print "\nCORRECTING QE IN " + inIm
# delete previous correction data and QE-corrected exposure
iraf.imdelete("q" + inIm)
iraf.imdelete("qecorr" + kwargs["refimages"])
iraf.gqecorr(inIm, **kwargs)
return
def rectifySpec(self, inIm, **kwargs):
# remove previous rectified spectra
iraf.imdelete("t" + inIm)
# transform the spectra and view the result
iraf.gftransform(inIm, **kwargs)
print "\nDisplaying 2D spectra"
iraf.display("t" + inIm + "[SCI]")
return
def create_coadd_img(qd,
targets,
dbFile,
data_dir,
prefix='mrg',
overwrite=True):
'''
despite the fact that it looks like this can be run from outside the
raw directory, it can't be.
Caveats:
* It takes a lot of patience and trial-and-error tweaking of parameters
to get good results
* There is little control over sky background
* The output image is no bigger than the first (reference) image,
rather than the union of the image footprints
'''
## Co-add the images, per position and filter.
print(" -- Begin image co-addition --")
cur_dir = os.getcwd()
iraf.chdir(data_dir)
# Use primarily the default task parameters.
gemtools.imcoadd.unlearn()
coaddFlags = {
'fwhm': 3,
'datamax': 6.e4,
'geointer': 'nearest',
'logfile': 'imcoaddLog.txt'
}
filters = ['Ha', 'HaC', 'SII', 'r', 'i']
for f in filters:
print " - Co-addding science images in filter: {}".format(f)
qd['Filter2'] = f + '_G%'
for t in targets:
qd['Object'] = t + '%'
print " - Co-addding science images for position: {}".format(t)
outImage = t + '_' + f + '.fits'
coAddFiles = fileSelect.fileListQuery(
dbFile, fileSelect.createQuery('sciImg', qd), qd)
if len(coAddFiles) > 1:
gemtools.imcoadd(','.join(prefix + str(x) for x in coAddFiles),
outimage=outImage,
**coaddFlags)
iraf.delete("*_trn*,*_pos,*_cen")
iraf.imdelete("*badpix.pl,*_med.fits,*_mag.fits")
#iraf.imdelete ("mrgS*.fits")
print("=== Finished Calibration Processing ===")
iraf.chdir(cur_dir)
def sumFibers(self, inIm, **kwargs):
print "\nSUMMING FIBERS FROM " + inIm
# delete previous summed spectra
# TODO: add attribute to class to turn imdelete verifications on/off
iraf.imdelete("a" + inIm)
# sum the fibers and view the final spectrum
iraf.gfapsum(inIm, **kwargs)
iraf.splot("a" + inIm + "[SCI,1]")
return
def extractSpec(self, inIm, **kwargs):
print "\nEXTRACTING SPECTRA FROM", inIm
# destroy previous extracted spectra
iraf.imdelete("e" + inIm)
# extract the spectra and view the result
iraf.gfextract(inIm, **kwargs)
self.viewCube("e" + inIm, frame=1, z1=0., z2=0., extname="SCI", \
version="*")
return
def viewWhiteIm(self, inIm, **kwargs):
print "\nDISPLAYING WHITE-LIGHT IMAGE"
idx = inIm.find(".")
#whiteIm = inIm[:idx] + "_2d.fits"
whiteIm = "whiteIm_" + inIm
iraf.imdelete(whiteIm)
iraf.imcombine(inIm + "[SCI]", whiteIm, project="yes", combine="sum", \
logfile="")
iraf.display(whiteIm)
return
def subSky(self, inIm, **kwargs):
print "\nSUBTRACTING SKY FROM " + inIm
# delete previous sky-subtracted spectra
iraf.imdelete("s" + inIm)
# subtract the sky and view the result (as image and datacube)
iraf.gfskysub(inIm, **kwargs)
print "\nDisplaying 2D spectra"
iraf.display("s" + inIm + "[SCI]")
print "\nDisplaying data cube"
self.viewCube("s" + inIm, extname="SCI", version="1")
return
def computeResp(self, inIm, **kwargs):
print "\nCOMPUTING RESPONSE FUNCTION FOR", inIm
# find endpoint of image prefix
prefEnd = inIm.find(".") - 14
# form title of output image and compute it with gfresponse
outIm = inIm[prefEnd:inIm.find(".")] + '_resp'
iraf.imdelete(outIm, verify="yes")
# compute and view response function and view it
iraf.gfresponse(inIm, outIm, **kwargs)
self.viewCube(outIm)
return
def stackCubes(self, cubeList, **kwargs):
outCube = self.target + "_finalCube.fits"
# extract filenames of individual data cubes
cubePaths = utils.getImPaths(cubeList)
cubes = ", ".join(cubePaths)
print "\nCOMBINING DATA CUBES: " + cubes
# make work directory and copy images there
cmd = "mkdir -v scratch/"
os.system(cmd)
for i, cube in enumerate(cubePaths):
cmd = "cp -v " + cube + " scratch/"
os.system(cmd)
cubePaths[i] = "scratch/" + cube
# update the WCS header cards of the cubes to their centroids
# TODO: allow for lower pixel limit?
# TODO: put calls to pyfalign and pyfmosaic into iterative loop?
pyfu.pyfalign(cubePaths, method="centroid", llimit=0)
# resample and align the cubes without stacking them
# TODO: add line below to view alignment
iraf.imdelete("scratch/separatedCubes.fits")
pyfu.pyfmosaic(cubePaths,
"scratch/separatedCubes.fits",
separate="yes")
# stack the aligned cubes and view the result
# (both as white-light image and slow-paced movie)
iraf.imdelete(outCube)
pyfu.pyfmosaic(cubePaths, outCube, propvar="yes")
self.viewWhiteIm(outCube)
time.sleep(30)
self.viewCubeMov(outCube)
# remove work directory and its contents
cmd = "rm scratch/*.fits"
os.system(cmd)
cmd = "rmdir scratch/"
os.system(cmd)
return
def reduceIm(self, inIm, **kwargs):
print "\nREDUCING", inIm
# remove previously reduced exposures
inPref = inIm[:inIm.find(".") - 14]
if inPref == "":
outPref = "g"
images = outPref + inIm
outPref = "r" + outPref
images += "," + outPref + inIm
else:
outPref = ""
images = ""
if "fl_gscrrej" in kwargs.keys() and kwargs["fl_gscrrej"] == "yes":
outPref = "x" + outPref
images += "," + outPref + inIm
else:
pass
if "fl_scatsub" in kwargs.keys() and kwargs["fl_scatsub"] == "yes":
outPref = "b" + outPref
images += "," + outPref + inIm
else:
pass
if "fl_qecorr" in kwargs.keys() and kwargs["fl_qecorr"] == "yes":
outPref = "q" + outPref
images += "," + outPref + inIm
else:
pass
if "fl_extract" in kwargs.keys() and kwargs["fl_extract"] == "no":
pass
else:
outPref = "e" + outPref
images += "," + outPref + inIm
iraf.imdelete(images)
# reduce the image
iraf.gfreduce(inIm, **kwargs)
return
def reduceBias(self, imList, outIm, **kwargs):
print "\nREDUCING BIASES"
# append prefix to file list
imList = "@" + imList
# remove previous master bias
iraf.imdelete(outIm)
iraf.gbias(imList, outIm, **kwargs)
# remove prepared images
junk = "g" + imList
iraf.imdelete(junk)
return
def diffDarOnOff(cleanDir1, cleanDir2):
files1tmp = glob.glob(cleanDir1 + '/c????.fits')
files2tmp = glob.glob(cleanDir2 + '/c????.fits')
for f1 in files1tmp:
cname1 = f1.split('/')[-1]
for f2 in files2tmp:
cname2 = f2.split('/')[-1]
if (cname1 == cname2):
outname = cname1.replace('c', 'diff')
print 'IMARITH: %s - %s = %s' % (cname1, cname2, outname)
if (os.path.exists(outname)):
iraf.imdelete(outname)
iraf.imarith(f1, '-', f2, outname)
def map_snr(waveLo=2.160, waveHi=2.175, datadir=datadir):
"""
Make SNR maps for the combo and sub-maps.
"""
fitsfiles = [datadir + cuberoot,
datadir + cuberoot + '_1',
datadir + cuberoot + '_2',
datadir + cuberoot + '_3']
for ff in range(len(fitsfiles)):
# Read in cube and cube image.
# - cube is indexed as [y, x, lambda]
# - cube iamge is indexed as [x, y]
cubefits = pyfits.open(fitsfiles[ff] + '.fits')
cube = cubefits[0].data
hdr = cubefits[0].header
cubeimg, imghdr = pyfits.getdata(fitsfiles[ff] + '_img.fits',
header=True)
# Calculate the SNR for the OSIRIS data between 2.160 and 2.175 microns.
# Get the wavelength solution so we can specify range:
w0 = hdr['CRVAL1']
dw = hdr['CDELT1']
wavelength = w0 + dw * np.arange(cube.shape[2], dtype=float)
wavelength /= 1000.0 # Convert to microns
# We need to remove the continuum first before we calculate the SNR.
widx = np.where((wavelength > 2.160) & (wavelength < 2.175))
waveCrop = wavelength[widx]
snrMap = np.zeros(cubeimg.shape, dtype=float)
# Loop through each spaxel and determine the SNR:
for xx in range(cubeimg.shape[0]):
for yy in range(cubeimg.shape[1]):
specCrop = cube[yy, xx, widx].flatten()
coeffs = scipy.polyfit(waveCrop, specCrop, 1)
residuals = specCrop / scipy.polyval(coeffs, waveCrop)
snrMap[xx, yy] = residuals.mean() / residuals.std()
snrFile = fitsfiles[ff] + '_snr.fits'
ir.imdelete(snrFile)
pyfits.writeto(snrFile, snrMap, header=imghdr)
def map_snr(waveLo=2.160, waveHi=2.175, datadir=datadir):
"""
Make SNR maps for the combo and sub-maps.
"""
fitsfiles = [datadir + cuberoot,
datadir + cuberoot + '_1',
datadir + cuberoot + '_2',
datadir + cuberoot + '_3']
for ff in range(len(fitsfiles)):
# Read in cube and cube image.
# - cube is indexed as [y, x, lambda]
# - cube iamge is indexed as [x, y]
cubefits = pyfits.open(fitsfiles[ff] + '.fits')
cube = cubefits[0].data
hdr = cubefits[0].header
cubeimg, imghdr = pyfits.getdata(fitsfiles[ff] + '_img.fits',
header=True)
# Calculate the SNR for the OSIRIS data between 2.160 and 2.175 microns.
# Get the wavelength solution so we can specify range:
w0 = hdr['CRVAL1']
dw = hdr['CDELT1']
wavelength = w0 + dw * np.arange(cube.shape[2], dtype=float)
wavelength /= 1000.0 # Convert to microns
# We need to remove the continuum first before we calculate the SNR.
widx = np.where((wavelength > 2.160) & (wavelength < 2.175))
waveCrop = wavelength[widx]
snrMap = np.zeros(cubeimg.shape, dtype=float)
# Loop through each spaxel and determine the SNR:
for xx in range(cubeimg.shape[0]):
for yy in range(cubeimg.shape[1]):
specCrop = cube[yy, xx, widx].flatten()
coeffs = scipy.polyfit(waveCrop, specCrop, 1)
residuals = specCrop / scipy.polyval(coeffs, waveCrop)
snrMap[xx, yy] = residuals.mean() / residuals.std()
snrFile = fitsfiles[ff] + '_snr.fits'
ir.imdelete(snrFile)
pyfits.writeto(snrFile, snrMap, header=imghdr)
def main():
parser = argparse.ArgumentParser(description="Convolve a higher resolution image to a lower resolution")
parser.add_argument("lopsf", type=str, help="Filename of lower res psf")
parser.add_argument("hipsf", type=str, help="Filename of higher res psf")
parser.add_argument("outker", type=str, help="Name of kernel to be created by psfmatch for convolution")
parser.add_argument(
"threshold",
type=float,
help='Low freq threshold in frac of total input image spectrum power for filtering option "replace"',
)
parser.add_argument("highresimg", type=str, help="Filename of image to be convolved")
parser.add_argument("outname", type=str, help="Filename of output, convolved image")
args = parser.parse_args()
lopsf = args.lopsf
hipsf = args.hipsf
outker = args.outker
threshold = args.threshold
highresimg = args.highresimg
outname = args.outname
padpsf = "padpsf"
if os.path.exists(padpsf + ".fits"):
imdelete(padpsf)
if os.path.exists(outker + ".fits"):
imdelete(outker)
# check that psfs are the same size
lnx, lny = fits.getheader(lopsf)["NAXIS1"], fits.getheader(lopsf)["NAXIS2"]
hnx, hny = fits.getheader(hipsf)["NAXIS1"], fits.getheader(hipsf)["NAXIS2"]
# the psfs should be square
if (lnx != lny) | (hnx != hny):
print "\npsfs not square."
exit()
if lnx != hnx:
# find which is smaller
print "\npsfs not equal size. Need to pad one with zeros."
print "Not implemented yet. ..."
exit()
else:
convolve(lopsf, hipsf, outker, threshold, highresimg, outname)
def mkIrafIm(self):
""" make the detection image. This method creates the detection
Image using the CHI^2 algorithm.
"""
self.logfile.write('Generating detection Image...')
cwd = os.getcwd()
os.chdir(self.obsFits) # cd to the observations Images dir
self.logfile.write('cd to observation Images dir ' + self.obsFits)
for i in range(len(self.statsList)):
image, background, noise = self.statsList[i]
expr = "((a - %s)/%s)**2" % (background, noise)
self.logfile.write(
'Generating background subtracted, squared image: ' +
'subtracted_' + image)
iraf.imexpr(expr, 'subtracted_' + image, a=image)
inputString = string.join(
map(lambda x: 'subtracted_' + x, self.sciImageList), ',')
iraf.unlearn(iraf.imsum)
iraf.imsum(input=inputString, output='dummy.fits', option='sum')
iraf.unlearn(iraf.imexpr)
self.logfile.write('Generating detectionImage.fits for observation...')
iraf.imexpr('sqrt(a)', self.detImName, a="dummy.fits")
self.logfile.write('Removing dummy image...')
iraf.imdelete('dummy.fits', verify='no')
self.logfile.write('Removing subtraction images...')
iraf.imdelete(inputString, verify='no')
# the header of the image produced by iraf is junk. fix it up
self._fixIrafHeader()
os.chdir(cwd) # return to orig dir.
return
def resampCube(self, inIm, **kwargs):
print "\nRESAMPLING DATA CUBE IN " + inIm
# determine name of output and delete previous version (if present)
if "outimage" in kwargs.keys():
outIm = kwargs["outimage"]
elif "outprefix" in kwargs.keys():
outIm = kwargs["outprefix"] + inIm
else:
outIm = "d" + inIm
iraf.imdelete(outIm)
# resample the data cube
iraf.gfcube(inIm, **kwargs)
# view white-light image
self.viewWhiteIm(outIm, **kwargs)
# view cube as movie in wavelength
self.viewCubeMov(outIm)
return
def clean_path(path, ext_list):
"""
Clean up files from path. If the path does not exist, it will be created. It's not recursive, so it
will not clean up a sub-folder within the path.
Args:
path (string): path to be cleaned.
ext_list (list): list of strings containing the extension of files to be deleted.
"""
path = str(os.path.join(path, ''))
if not os.path.exists(path):
log.info('Provided path does not exist, but it was created!')
os.makedirs(path)
elif os.path.exists(path):
log.info('Cleaning up ' + path + '.')
iraf.imdelete('*tmp*', verify='no')
for ext in ext_list:
for _file in glob.glob(os.path.join(path, '*.' + str(ext))):
os.remove(_file)
def interp_single_spec(i, lines):
filename = lines[i].strip()
#get wavelength solution for this file
disp_new = ns.getdisp(_wldat + '/ec' + filename)
w_new = ns.dispeval(disp_new[0],
disp_new[1],
disp_new[2],
shift=disp_new[3])
w_new = w_new[::-1]
# label wavelength and save to file
# upsample data if that option is passed
ns.interp_spec(filename,
w_new,
w_interp,
interp=upsampleData,
k=3.0,
suffix=interp_suffix,
badval=badval,
clobber=True,
verbose=False)
if saveUnInterpolated == False:
ir.imdelete(filename)
def call_lacos(args, science, Nslits=0, longslit=False):
outfile = '{0}_lacos.fits'.format(science)
#if os.path.isfile(outfile):
#os.remove(outfile)
if utils.skip(args, 'lacos', outfile):
return outfile[:-5]
print()
print('-' * 30)
print()
print('Removing cosmic rays with LACos ...')
to = time()
head = pyfits.getheader(science + '.fits')
gain = head['GAIN']
rdnoise = head['RDNOISE']
#utils.delete(outfile)
if os.path.isfile(outfile):
iraf.imdelete(outfile)
os.system('cp -p ' + science + '.fits ' + outfile)
utils.removedir('slits')
utils.makedir('slits')
iraf.imcopy.unlearn()
if longslit:
slit = '{0}[sci,1]'.format(science)
outslit = os.path.join('slits' '{0}_long'.format(science))
outmask = os.path.join('slits' '{0}_longmask'.format(science))
iraf.lacos_spec(slit, outslit, outmask, gain=gain, readn=rdnoise)
iraf.imcopy(outslit, '{0}[SCI,1,overwrite]'.format(outfile[:-5]),
verbose='no')
else:
for i in xrange(1, Nslits+1):
slit = '{0}[sci,{1}]'.format(science, i)
print('slit =', slit)
outslit = os.path.join('slits', '{0}_{1}'.format(science, i))
outmask = os.path.join('slits', '{0}_mask{1}'.format(science, i))
if os.path.isfile(outslit):
iraf.imdelete(outslit)
if os.path.isfile(outmask):
iraf.imdelete(outmask)
iraf.lacos_spec(slit, outslit, outmask, gain=gain, readn=rdnoise)
iraf.imcopy(
outslit, '{0}[SCI,{1},overwrite]'.format(outfile[:-5], i),
verbose='yes')
utils.delete('lacos*')
utils.removedir('slits')
print(outfile[:-5])
print('Done in {0:.2f}'.format((time()-to)/60))
print()
print('-' * 30)
print()
return outfile[:-5]
hdulist.writeto(filename+'_Handmask.fits')
#Do the same for the Ha image
print 'Writing Ha mask'
try:
maskedHaImage = fits.getdata(filename+'_HaHandmasked.fits')
except IOError:
maskedHaImage = fits.getdata(filename+'_Ha.fits')
HaMaskPixels = np.where((maskedHaImage!=maskValue),manyZeros,manyOnes)
hdu=fits.PrimaryHDU(HaMaskPixels)
hdulist = fits.HDUList([hdu])
hdulist.writeto(filename+'_HaHandmask.fits')
######################
filename = sys.argv[1]
maskReversed=False
try:
with open(filename+'MaskReversed.fits'):maskReversed=True
except IOError:
print 'Reversing input mask'
if(not maskReversed):
reverseMask(filename)
maskImages(filename)
getMasks(filename)
print('Removing superfluous files...')
iraf.imdelete(filename+'MaskReversed.fits')
print('Done!')
def cleanup(self):
if not self.failed:
iraf.imdelete(self.testfile, verify=iraf.no)
def solveField(fullfilename, findstarmethod="astrometry.net"):
"""
@param: fullfilename entire path to image
@type: str
@param: findstarmethod (astrometry.net, sex)
@type: str
Does astrometry to image=fullfilename
Uses either astrometry.net or sex(tractor) as its star finder
"""
pathname, filename = os.path.split(fullfilename)
pathname = pathname + "/"
basefilename, file_xtn = os.path.splitext(filename)
# *** enforce .fits extension
if (file_xtn != ".fits"):
raise ValueError(
"File extension must be .fits it was = %s\n" % file_xtn)
# *** check whether the file exists or not
if (os.path.exists(fullfilename) == False):
raise IOError(
"You selected image %s It does not exist\n" % fullfilename)
# version 0.23 changed behavior of --overwrite
# I need to specify an output filename with -o
outfilename = basefilename + "-out"
image = Image.fromFile(fullfilename)
try:
ra = image["CRVAL1"] # expects to see this in image
except:
raise AstrometryNetException(
"Need CRVAL1 and CRVAL2 and CD1_1 on header")
try:
dec = image["CRVAL2"]
except:
raise AstrometryNetException(
"Need CRVAL1 and CRVAL2 and CD1_1 on header")
width = image["NAXIS1"]
height = image["NAXIS2"]
radius = 5.0 * abs(image["CD1_1"]) * width
if findstarmethod == "astrometry.net":
line = "solve-field %s -d 10,20,30,40,50,60,70,80,90,100 --overwrite -o %s --ra %f --dec %f --radius %f" % (
fullfilename, outfilename, ra, dec, radius)
elif findstarmethod == "sex":
sexoutfilename = pathname + outfilename + ".xyls"
line = "solve-field %s -d 10,20,30,40,50,60,70,80,90,100 --overwrite -o %s --x-column X_IMAGE --y-column Y_IMAGE --sort-column MAG_ISO --sort-ascending --width %d --height %d --ra %f --dec %f --radius %f" % (
sexoutfilename, outfilename, width, height, ra, dec, radius)
# line = "solve-field %s --overwrite -o %s --x-column X_IMAGE --y-column Y_IMAGE --sort-column MAG_ISO --sort-ascending --width %d --height %d" %(sexoutfilename, outfilename,width, height)
# could use --guess-scale for unreliable mounts:
# line = "solve-field %s --overwrite -o %s --x-column X_IMAGE --y-column Y_IMAGE --sort-column MAG_ISO --sort-ascending --width %d --height %d --guess-scale" %(sexoutfilename, outfilename, width, height)
sex = SExtractor()
sex.config['BACK_TYPE'] = "AUTO"
sex.config['DETECT_THRESH'] = 3.0
sex.config['DETECT_MINAREA'] = 18.0
sex.config['VERBOSE_TYPE'] = "QUIET"
sex.config['CATALOG_TYPE'] = "FITS_1.0"
#sex.config['CATALOG_TYPE'] = "ASCII"
sex.config['CATALOG_NAME'] = sexoutfilename
sex.config['PARAMETERS_LIST'] = ["X_IMAGE", "Y_IMAGE", "MAG_ISO"]
sex.run(fullfilename)
else:
log.error("Unknown option used in astrometry.net")
# when there is a solution astrometry.net creates a file with .solved
# added as extension.
is_solved = pathname + outfilename + ".solved"
# if it is already there, make sure to delete it
if (os.path.exists(is_solved)):
os.remove(is_solved)
print "SOLVE", line
# *** it would be nice to add a test here to check
# whether astrometrynet is running OK, if not raise a new exception
# like AstrometryNetInstallProblem
solve = Popen(line.split()) # ,env=os.environ)
solve.wait()
# if solution failed, there will be no file .solved
if (os.path.exists(is_solved) == False):
raise NoSolutionAstrometryNetException(
"Astrometry.net could not find a solution for image: %s %s" % (fullfilename, is_solved))
# wcs_imgname will be the old fits file with the new header
# wcs_solution is the solve-field solution file
wcs_imgname = pathname + outfilename + "-wcs" + ".fits"
wcs_solution = pathname + outfilename + ".wcs"
shutil.copyfile(wcs_solution, wcs_solution + ".fits")
if (os.path.exists(wcs_imgname) == True):
iraf.imdelete(wcs_imgname)
# create a separate image with new header
iraf.artdata()
iraf.imcopy(fullfilename, wcs_imgname)
iraf.hedit(wcs_imgname, "CD1_1,CD1_2,CD2_1,CD2_2,CRPIX1,CRPIX2,CRVAL1,CRVAL2,RA,DEC,ALT,AZ",
add="no", addonly="no", delete="yes",
verify="no", show="no", update="yes")
iraf.mkheader(images=wcs_imgname, headers=wcs_solution + ".fits",
append="yes", verbose="no", mode="al")
return(wcs_imgname)
#!/usr/bin/env python
import sys,os,string
import pyraf
from pyraf import iraf
from iraf import images,tv,sleep,imutil,imgeom,blkrep,imcopy,imdelete
imagen=sys.argv[1]
iraf.imcopy(input=imagen+'[1]',output="test.fits")
iraf.blkrep(input="test.fits",output=imagen+'_GMOS',b1=12,b2=1)
for i in range (1,13):
# print i
if i < 2:
# print i
iraf.imcopy(input=imagen+'['+str(i)+']',output=imagen+'_GMOS.fits[1:'+str(288)+',*]')
# if i>1 and i<12:
else:
# print i,i
iraf.imcopy(input=imagen+'['+str(i)+']',output=imagen+'_GMOS.fits['+str((i-1)*288)+':'+str((i)*288)+',*]')
# if i>12:
# iraf.imcopy(input=imagen+'['+str(i)+']',output=imagen+'_GMOS.fits['+str(i*288)+':'+str((i+1)*288)+',*]')
iraf.imdelete("test.fits")
def cleanup(self):
if not self.failed:
iraf.imdelete(self.testfile, verify=iraf.no)
def osiris_performance(cubefile, rootdir=datadir, plotdir=workdir):
"""
Determine the spatial resolution of an OSIRIS data cube image by
comparing with a NIRC2 image. The NIRC2 image is convolved with a
gaussian, rebinned to the OSIRIS plate scale and differenced until
the optimal gaussian width is determined.
"""
# Read in the cube... assume it is K-band
cube, cubehdr = pyfits.getdata(rootdir + cubefile, header=True)
cubeimg = make_cube_image(cubefile, rootdir=rootdir)
if cubeimg == None:
print 'Opening previosly existing cube image.'
cubeimg = pyfits.getdata(rootdir + cubefile.replace(".fits", '_img.fits'))
### Register the NIRC2 image to the OSIRIS image.
# Read in the NIRC2 image (scale = 10 mas/pixel)
nirc2file = '/u/jlu/data/m31/05jul/combo/m31_05jul_kp.fits'
img, imghdr = pyfits.getdata(nirc2file, header=True)
# Get the PA of the OSIRIS spectrograph image
specPA = cubehdr['PA_SPEC']
# Get the PA of the NIRC2 image
imagPA = imghdr['ROTPOSN'] - imghdr['INSTANGL']
# Rotate the NIRC2 image
angle = specPA - imagPA
imgrot = scipy.misc.imrotate(img, angle, interp='bicubic')
# Get the shifts constructed manually
xshift = 0
yshift = 0
shiftsTable = asciidata.open(rootdir + 'shifts.txt')
for rr in range(shiftsTable.nrows):
if rr == 0:
xshift0 = float(shiftsTable[1][rr])
yshift0 = float(shiftsTable[2][rr])
if shiftsTable[0][rr] == cubefile:
xshift = float(shiftsTable[1][rr])
yshift = float(shiftsTable[2][rr])
print 'Shifting image: '
print shiftsTable[0][rr], shiftsTable[1][rr], shiftsTable[2][rr]
print ''
break
# Calculate the SNR for the OSIRIS data between 2.160 and 2.175 microns.
# This is done for the same position in each of the combo + subset mosaics.
# We need to remove the continuum first before we calculate the SNR.
xpixSNR = 11 + (xshift - xshift0)
ypixSNR = 47 + (yshift - yshift0)
# if (xpixSNR < 0 or xpixSNR >= cube.shape[0] or
# ypixSNR < 0 or ypixSNR >= cube.shape[1]):
tmp = np.unravel_index(cubeimg.argmax(), cubeimg.shape)
xpixSNR = tmp[1]
ypixSNR = tmp[0]
# Get the wavelength solution so we can specify range:
w0 = cubehdr['CRVAL1']
dw = cubehdr['CDELT1']
wavelength = w0 + dw * np.arange(cube.shape[2], dtype=float)
wavelength /= 1000.0 # Convert to microns
widx = np.where((wavelength > 2.160) & (wavelength < 2.175))
waveCrop = wavelength[widx]
specCrop = cube[xpixSNR, ypixSNR, widx].flatten()
coeffs = scipy.polyfit(waveCrop, specCrop, 1)
residuals = specCrop / scipy.polyval(coeffs, waveCrop)
specSNR = (residuals.mean() / residuals.std())
print 'OSIRIS Signal-to-Noise:'
print ' X = %d Y = %d' % (xpixSNR, ypixSNR)
print ' wavelength = [%5.3f - %5.3f]' % (2.160, 2.175)
print ' SNR = %f' % specSNR
# Trim down the NIRC2 image to the same size as the OSIRIS image.
# Still bigger than OSIRIS FOV
ycent = int(round((img.shape[0] / 2.0) + (yshift*5.0)))
xcent = int(round((img.shape[1] / 2.0) - (xshift*5.0)))
print ''
print 'Comparing OSIRIS with NIRC2 Image:'
print ' NIRC2 xcent = ', xcent, ' ycent = ', ycent
yhalf = int(cubeimg.shape[0] / 2) * 5 # Make the image the same size
xhalf = int(cubeimg.shape[1] / 2) * 5 # as the OSIRIS cube image.
img = imgrot[ycent-yhalf:ycent+yhalf, xcent-xhalf:xcent+xhalf]
# Rebin the NIRC2 image to the same 50 mas plate scale as the
# OSIRIS image.
img = scipy.misc.imresize(img, 0.2) # rebin to 50 mas/pixel.
# Save the modified NIRC2 image.
nirc2_file = rootdir + 'nirc2_ref.fits'
ir.imdelete(nirc2_file)
pyfits.writeto(nirc2_file, img, header=imghdr, output_verify='silentfix')
# Clean up the cube image to get rid of very very low flux values
# (on the edges).
cidx = np.where(cubeimg < cubeimg.max()*0.05)
cubeimg[cidx] = 0
def fitfunction(params, plot=False, verbose=False):
amp1 = abs(params[0])
amp2 = abs(params[1])
sigma1 = abs(params[2])
sigma2 = abs(params[3])
# Actually amp1 should be fixed to 1.0
amp1 = 1.0
sigma2 = 8.0
# Convolve the NIRC2 image with a gaussian
boxsize = min(cubeimg.shape) / 2
psf = twogauss_kernel(sigma1, sigma2, amp1, amp2, half_box=50)
newimg = signal.convolve(img, psf, mode='full')
xlo = (psf.shape[1] / 2)
xhi = xlo + cubeimg.shape[1]
ylo = (psf.shape[0] / 2)
yhi = ylo + cubeimg.shape[0]
newimg = newimg[ylo:yhi, xlo:xhi]
if verbose:
print 'fitfunc shapes:',
print ' cubeimg = ', cubeimg.shape,
print ' psf = ', psf.shape,
print ' newimg = ', newimg.shape
newimg[cidx] = 0
cubeimg_norm = cubeimg / cubeimg.sum()
newimg_norm = newimg / newimg.sum()
residuals = (cubeimg_norm - newimg_norm) / np.sqrt(cubeimg_norm)
residuals[cidx] = 0
if verbose:
print 'Parameters: sig1 = %5.2f sig2 = %5.2f ' % (sigma1, sigma2),
print ' amp1 = %9.2e amp2 = %9.2e' % (amp1, amp2)
print 'Residuals: ', math.sqrt((residuals*residuals).sum())
print ''
if plot:
py.figure(1)
py.clf()
py.subplots_adjust(left=0.05, right=0.95, bottom=0.05, top=0.95)
py.subplot(1, 3, 1)
py.imshow(cubeimg)
py.title('OSIRIS')
py.subplot(1, 3, 2)
py.imshow(newimg)
py.title('NIRC2+')
py.subplot(1, 3, 3)
py.imshow(residuals)
py.title('Residuals')
py.savefig(plotdir + 'osir_perf_' +
cubefile.replace('.fits', '.png'))
py.show()
return residuals.flatten()
params = np.zeros(4, dtype=float)
params[0] = 1.0 # amp1
params[1] = 0.000001 # amp2
params[2] = 0.9 # sigma1 (near-diffraction-limit)
params[3] = 6.0 # sigma2 (seeing halo)
print 'Fitting PSF: '
print ''
print 'Initial: '
fitfunction(params, plot=True, verbose=True)
p, cov, infodict, errmsg, success = scipy.optimize.leastsq(fitfunction,
params,
full_output=1,
maxfev=100)
print ''
print 'Results:'
residuals = fitfunction(p, plot=True, verbose=True)
amp1 = abs(p[0])
amp2 = abs(p[1])
sigma1 = abs(p[2])
sigma2 = abs(p[3])
_out = open(plotdir + 'osir_perf_' + cubefile.replace('.fits', '_params.txt'), 'w')
_out.write('sig1: %5.2f sig2: %5.2f amp1: %9.2e amp2: %9.2e res: %7.5f ' %
(sigma1, sigma2, amp1, amp2, math.sqrt((residuals**2).sum())))
_out.write('xpixSNR: %2d ypixSNR: %2d SNR: %5.1f\n' % (xpixSNR, ypixSNR, specSNR))
_out.close()
def execute():
################NGC1023####################################################
## use GALFIT to fit and to create a model disk vs bulge##
##galfit.feedme##
iraf.images()
# path to fits files:
total = config.iraf_input_dir+config.totalfits
bulge = config.iraf_input_dir+config.bulgefits
fraction = config.iraf_tmp_dir+config.fractionfits
input_dir = config.iraf_input_dir
tmp_dir = config.iraf_tmp_dir
if not os.path.exists(tmp_dir):
os.mkdir(tmp_dir)
if os.path.exists(fraction):
iraf.imdelete(fraction)
print fraction
print bulge
# Makes the fraction image by dividing the bulge by the total:
iraf.imarith(bulge, "/", total, fraction)
""" LLT: I don't think we need this anymore:
if os.path.exists(tmp_dir+"snb.fits"):
iraf.module.imdelete(tmp_dir+"snb.fits")
if os.path.exists(tmp_dir+"tnb.fits"):
iraf.module.imdelete(tmp_dir+"tnb.fits")
iraf.module.imcopy(bulge, tmp_dir+"snb.fits")
iraf.module.imcopy(total, tmp_dir+"tnb.fits")
"""
##########################################################################
##Note: the coordinate in pixel as obtained from RA and Dec differ from the
##pixels one because of distorsion corrections as done with astrometry==>
##RA and Dec coordinate in arcsec are the correct one and don't need do be
##transfer in pixels
############################################################################
##Consistency check
#displ(image="../fits_inputs/galaxy.fits",frame=1)
#tvmark(frame=1,coords="Kall.dat",color=205,lab-,num+)
#displ(image="f.fits",zrange=no,zscale=no,ztrans="linear",z1=0,z2=1,frame=3)
#tvmark(frame=3,coords="Kall.dat",color=201,mark="rectangle",lengths="6",lab-,num-)
###########################################################################
##Use program overplot.sm to create both Kall.dat & compagna.dat
##in the RA & Dec of PNS there is a rotation of 90deg respect to the pixel
## x-->-x
##after long reflexion I decide the right PA is 90-47.11=42.89 (just turning the PNe as observed on the R image)
##########################################################################
############################################################################
## to measure the relative flux use phot (daophot) ##with centerpars=none!!!###
##I used sigma=stdv(skys)=0 ===> no bg
##fwhmpsf=2.27 as in galfit psf2.fits (imexam A enclosed)
##skyvalue=0 ===> no bg
##no centerpars
##apertures=3.0 pixels (=1.8 arcsec)
##zmag=25
##snb.fits & tnb.fits without bg (when I made the model)
# filenames
silentremove(config.bulgemag)
silentremove(config.bulgetxt)
silentremove(config.totalmag)
silentremove(config.totaltxt)
silentremove(config.fractionmag)
silentremove(config.fractiontxt)
iraf.noao(_doprint=False)
iraf.digiphot(_doprint=False)
iraf.daophot(_doprint=False)
iraf.phot(bulge,config.position_file, config.bulgemag, skyfile="", datapars="",scale=1.,fwhmpsf=2.27,emissio="yes",sigma=0.0,datamin=-100, datamax=1e5,noise="poisson", centerpars="",calgorithm="none",cbox=0,cthreshold=0.,minsnratio=1., cmaxiter=0,maxshift=0,clean="no",rclean=0.,rclip=0.,kclean=0., mkcenter="no", fitskypars="",salgorithm="constant",annulus=10,dannulus=10,skyvalue=0., smaxiter=10,sloclip=0.,shiclip=0.,snreject=50,sloreject=3., shireject=3.,khist=3.,binsize=0.1,smooth="no",rgrow=0., mksky="no", photpars="",weighting="constant",apertures=3,zmag=0.,mkapert="no", interactive="no",radplots="no",verify="no",update="no",verbose="no", graphics="stdgraph",display="stdimage",icommands="",gcommands="")
# LLT: Corrected to pyraf, use stdout = instead of >>
iraf.txdump(config.bulgemag, "XCENTER,YCENTER,STDEV,FLUX,SUM,AREA,ID", "yes", headers="no",paramet="no", Stdout=config.bulgetxt)
iraf.phot(total, config.position_file, config.totalmag, skyfile="", datapars="",scale=1.,fwhmpsf=2.27,emissio="yes",sigma=0.0,datamin=-100, datamax=1e5,noise="poisson", centerpars="",calgorithm="none",cbox=0,cthreshold=0.,minsnratio=1., cmaxiter=0,maxshift=0.,clean="no",rclean=0.,rclip=0.,kclean=0., mkcenter="no", fitskypars="",salgorithm="constant",annulus=10,dannulus=10,skyvalue=0., smaxiter=10,sloclip=0.,shiclip=0.,snreject=50,sloreject=3., shireject=3.,khist=3.,binsize=0.1,smooth="no",rgrow=0., mksky="no", photpars="",weighting="constant",apertures=3,zmag=25.,mkapert="no", interactive="no",radplots="no",verify="no",update="no",verbose="no", graphics="stdgraph",display="stdimage",icommands="",gcommands="")
iraf.txdump(config.totalmag, "XCENTER,YCENTER,STDEV,FLUX,SUM,AREA,ID", "yes", headers="no",paramet="no", Stdout=config.totaltxt)
iraf.phot(fraction,config.position_file, config.fractionmag, skyfile="", datapars="", scale=1., fwhmpsf=2.27, emissio="no", sigma=0.0, datamin=-100, datamax=1e5, noise="poisson", centerpars="", calgorithm="none", cbox=0, cthreshold=0., minsnratio=1., cmaxiter=0, maxshift=0., clean="no", rclean=0., rclip=0., kclean=0., mkcenter="no", fitskypars="", salgorithm="constant", annulus=10, dannulus=10, skyvalue=0., smaxiter=10, sloclip=0., shiclip=0., snreject=50, sloreject=3., shireject=3., khist=3., binsize=0.1, smooth="no", rgrow=0., mksky="no", photpars="", weighting="constant", apertures=3, zmag=25., mkapert="no", interactive="no", radplots="no", verify="no", update="no", verbose="no", graphics="stdgraph", display="stdimage", icommands="", gcommands="")
iraf.txdump(config.fractionmag, "XCENTER,YCENTER,STDEV,FLUX,SUM,AREA,ID", "yes", headers="no",paramet="no", Stdout=config.fractiontxt)
# Set detector properties:
gain = 4.0 # photons (i.e., electrons) per data unit
readnoise = 10.0 # photons (i.e., electrons)
ir.imcombine.gain = gain
ir.imcombine.rdnoise = readnoise
ir.apall.gain = gain
ir.apall.readnoise = readnoise
ir.apnormalize.gain = gain
ir.apnormalize.readnoise = readnoise
ir.set(observatory=observ)
# Combine dark frames into a single dark frame:
if makeDark:
ir.imdelete(_sdark)
ir.imdelete(_sdarks)
print "rawdark file list>>" + rawdark
ir.imcombine("@" + rawdark,
output=_sdark,
combine="average",
reject="avsigclip",
sigmas=_sdarks,
scale="none",
weight="none",
bpmasks="")
ns.write_exptime(_sdark, itime=itime)
if makeFlat: # 2008-06-04 09:21 IJC: dark-correct flats; then create super-flat
def makesky_fromsci(files, nite, wave):
"""Make short wavelength (not L-band or longer) skies."""
# Start out in something like '06maylgs1/reduce/kp/'
waveDir = os.getcwd() + '/'
redDir = util.trimdir(os.path.abspath(waveDir + '../') + '/')
rootDir = util.trimdir(os.path.abspath(redDir + '../') + '/')
skyDir = waveDir + 'sky_' + nite + '/'
rawDir = rootDir + 'raw/'
util.mkdir(skyDir)
print 'sky dir: ',skyDir
print 'wave dir: ',waveDir
skylist = skyDir + 'skies_to_combine.lis'
output = skyDir + 'sky_' + wave + '.fits'
util.rmall([skylist, output])
nn = [skyDir + 'n' + str(i).zfill(4) for i in files]
nsc = [skyDir + 'scale' + str(i).zfill(4) for i in files]
skies = [rawDir + 'n' + str(i).zfill(4) for i in files]
for ii in range(len(nn)):
ir.imdelete(nn[ii])
ir.imdelete(nsc[ii])
ir.imcopy(skies[ii], nn[ii], verbose="no")
# Make list for combinng. Reset the skyDir to an IRAF variable.
ir.set(skydir=skyDir)
f_on = open(skylist, 'w')
for ii in range(len(nn)):
nn_new = nn[ii].replace(skyDir, "skydir$")
f_on.write(nn_new + '\n')
f_on.close()
# Calculate some sky statistics, but reject high (star-like) pixels
sky_mean = np.zeros([len(skies)], dtype=float)
sky_std = np.zeros([len(skies)], dtype=float)
text = ir.imstat("@" + skylist, fields='midpt,stddev', nclip=10,
lsigma=10, usigma=3, format=0, Stdout=1)
for ii in range(len(nn)):
fields = text[ii].split()
sky_mean[ii] = float(fields[0])
sky_std[ii] = float(fields[1])
sky_mean_all = sky_mean.mean()
sky_std_all = sky_std.mean()
# Upper threshold above which we will ignore pixels when combining.
hthreshold = sky_mean_all + 3.0 * sky_std_all
ir.imdelete(output)
ir.unlearn('imcombine')
ir.imcombine.combine = 'median'
ir.imcombine.reject = 'sigclip'
ir.imcombine.mclip = 'yes'
ir.imcombine.hsigma = 2
ir.imcombine.lsigma = 10
ir.imcombine.hthreshold = hthreshold
ir.imcombine('@' + skylist, output)
def map_co_eqw():
# Read in cube and cube image.
# - cube is indexed as [y, x, lambda]
# - cube iamge is indexed as [x, y]
cubefits = pyfits.open(datadir + cuberoot + '.fits')
cube = cubefits[0].data
cubehdr = cubefits[0].header
cubeNoise = cubefits[1].data
img, imghdr = pyfits.getdata(datadir + cuberoot + '_img.fits', header=True)
# Get the wavelength solution.
w0 = cubehdr['CRVAL1']
dw = cubehdr['CDELT1']
wave = w0 + dw * np.arange(cube.shape[2], dtype=float)
wave /= 1000.0 # Convert to microns
##########
# Using the CO index from Marmol-Queralto et al. 2008.
# Wiki Notes:
# http://131.215.103.243/groups/jlu/wiki/65bbf/M31__Line_Maps.html
##########
# Define the wavelength regions for continuum #1, continuum #2, absorption
waveReg = np.array([[2.2460, 2.2550],
[2.2710, 2.2770],
[2.2880, 2.3010]])
# Redshift the regions for M31's systemic velocity
m31rv = -300.99 # km/s
waveReg *= (1.0 + m31rv / 2.99792e5)
# Continuum region #1
idxC1 = np.where((wave >= waveReg[0,0]) & (wave <= waveReg[0,1]))[0]
waveC1 = wave[idxC1]
# Continuum region #2
idxC2 = np.where((wave >= waveReg[1,0]) & (wave <= waveReg[1,1]))[0]
waveC2 = wave[idxC2]
# CO region
idxA1 = np.where((wave >= waveReg[2,0]) & (wave <= waveReg[2,1]))[0]
waveA1 = wave[idxA1]
# Loop through each pixel and calculate the CO index.
coMap = np.zeros(img.shape, dtype=float)
coMapErr = np.zeros(img.shape, dtype=float)
contMap = np.zeros(img.shape, dtype=float)
py.clf()
for yy in range(img.shape[0]):
for xx in range(img.shape[1]):
fluxC1 = cube[xx, yy, idxC1].sum()
fluxC2 = cube[xx, yy, idxC2].sum()
fluxA1 = cube[xx, yy, idxA1].sum()
if yy == 46 and xx == 10:
py.plot(waveC1, cube[xx, yy, idxC1], 'b-')
py.plot(waveC2, cube[xx, yy, idxC2], 'b-')
py.semilogy(waveA1, cube[xx, yy, idxA1], 'b-')
py.title('xx=10, yy=46')
if yy == 35 and xx == 15:
py.plot(waveC1, cube[xx, yy, idxC1], 'g-')
py.plot(waveC2, cube[xx, yy, idxC2], 'g-')
py.plot(waveA1, cube[xx, yy, idxA1], 'g-')
py.title('xx=15, yy=35')
# Need to filter out bad pixel issues in noise map...
# just use the median error instead.
#varC1 = (cube[xx, yy, idxC1].std())**2
varC1 = np.median(cubeNoise[xx, yy, idxC1])**2
varC1 *= float(len(idxC1))
#varC2 = (cube[xx, yy, idxC2].std())**2
varC2 = np.median(cubeNoise[xx, yy, idxC2])**2
varC2 *= float(len(idxC2))
#varA1 = (cube[xx, yy, idxC2].std())**2 # note same as C2
varA1 = np.median(cubeNoise[xx, yy, idxA1])**2
varA1 *= float(len(idxA1))
dwaveC1 = waveReg[0,1] - waveReg[0,0]
dwaveC2 = waveReg[1,1] - waveReg[1,0]
dwaveA1 = waveReg[2,1] - waveReg[2,0]
contFlux = (fluxC1 + fluxC2) / ( dwaveC1 + dwaveC2)
absoFlux = fluxA1 / dwaveA1
contFluxVar = (varC1 + varC2) / (dwaveC1 + dwaveC2)**2
absoFluxVar = varA1 / dwaveA1**2
coMap[yy, xx] = contFlux / absoFlux
coMapErr[yy, xx] = math.sqrt(((contFlux**2 * absoFluxVar) +
(absoFlux**2 * contFluxVar))
/ absoFlux**4)
contMap[yy, xx] = (fluxC1 / dwaveC1) / (fluxC2 / dwaveC2)
py.show()
coFile = workdir + 'maps/co_map.fits'
coErrFile = workdir + 'maps/co_err_map.fits'
contFile = workdir + 'maps/cont_ratio_map.fits'
ir.imdelete(coFile)
ir.imdelete(coErrFile)
ir.imdelete(contFile)
pyfits.writeto(coFile, coMap, header=imghdr)
pyfits.writeto(coErrFile, coMapErr, header=imghdr)
pyfits.writeto(contFile, contMap, header=imghdr)
# Lets plot the CO index in various ways
x1d = np.arange(img.shape[1])
y1d = np.arange(img.shape[0])
y2d, x2d = scipy.mgrid[0:img.shape[0], 0:img.shape[1]]
py.clf()
py.errorbar(x2d.flatten(), coMap.flatten(), yerr=coMapErr.flatten(),
fmt='.')
#py.show()
py.clf()
py.errorbar(y2d.flatten(), coMap.flatten(), yerr=coMapErr.flatten(),
fmt='.')
#py.show()
# Calculate the mean CO index for the whole map. Then make
# a map of the significance of the excess CO at each pixel
idx = np.where(coMap > 0) # good pixels
coAvg = coMap[idx[0],idx[1]].mean()
coSigma = (coMap - coAvg) / coMapErr
coSigmaMasked = np.ma.masked_invalid(coSigma)
coSigFile = workdir + 'maps/co_sigma_map.fits'
ir.imdelete(coSigFile)
pyfits.writeto(coSigFile, coSigma, header=imghdr)
py.clf()
py.imshow(coSigmaMasked)
py.colorbar(orientation='vertical')
py.show()
'fl_paste':'no','fl_fixpix':'no','fl_clean':'yes','geointer':'nearest',
'logfile':'gmosaicLog.txt','fl_vardq':'yes','fl_fulldq':'yes','verbose':'no'
}
# Reduce the science images, then mosaic the extensions in a loop
for f in filters:
print " Processing science images for: %s" % (f)
qd['Filter2'] = f + '_G%'
flatFile = 'MCflat_' + f + '.fits'
sciFiles = fs.fileListQuery(dbFile, fs.createQuery('sciImg', qd), qd)
if len(sciFiles) > 0:
gmos.gireduce (','.join(str(x) for x in sciFiles), bias='MCbias',
flat1=flatFile, **sciFlags)
for file in sciFiles:
gmos.gmosaic (prefix+file, **mosaicFlags)
iraf.imdelete('gS2006*.fits,rgS2006*.fits')
## Co-add the images, per position and filter.
print (" -- Begin image co-addition --")
# Use primarily the default task parameters.
gemtools.imcoadd.unlearn()
coaddFlags = {
'fwhm':3,'datamax':6.e4,'geointer':'nearest','logfile':'imcoaddLog.txt'
}
targets = ['M8-1', 'M8-2', 'M8-3']
prefix = 'mrg'
for f in filters:
print " - Co-addding science images in filter: %s" % (f)
qd['Filter2'] = f + '_G%'
for t in targets:
def makeResolutionMapCO():
"""
Use a specially created M31 mosaic where the sky lines have not been
removed. Determine the spectral resolution at each spaxal from 3
sky lines around 2.2 microns. This should be the same resolution
for all wavelengths > 2.2 microns based on some figures in the OSIRIS
manual.
"""
# Here are the 3 OH sky lines we will be using. Wavelengths were
# extractged from the ohlines.dat file included with IRAF:
# /Applications/scisoft/all/Packages/iraf/iraf/noao/lib/linelists/
lines = np.array([2.19556, 2.21255, 2.23127])
# Old 2008 analysis
#cubefile = datadir + '../noss/' + cuberoot + '.fits'
# New 2010 analysis
cubefile = datadir2010 + '../100829/SPEC/reduce/sky/sky_900s_cube_nodrk_tlc.fits'
# Read in the cube... assume it is K-band
cube, cubehdr = pyfits.getdata(cubefile, header=True)
# Get the wavelength solution so we can specify range:
w0 = cubehdr['CRVAL1']
dw = cubehdr['CDELT1']
wavelength = w0 + dw * np.arange(cube.shape[2], dtype=float)
wavelength /= 1000.0 # Convert to microns
# Gaussian line fits
halfWindow = dw * 7.0 / 1000.0
def gaussian(p, x):
amp, sigma, center, constant = p
y = amp * np.exp(-(x-center)**2 / (2*sigma**2))
y += constant
return y
def fitfunc(p, wavelength, flux, line):
#p[2] = line
model = gaussian(p, wavelength)
residuals = flux - model
return residuals
# Recall that the images are transposed relative to the cubes.
resolutionMap = np.zeros((cube.shape[1], cube.shape[0]), dtype=float)
ampAll = np.zeros((cube.shape[1], cube.shape[0], 3), dtype=float)
resAll = np.zeros((cube.shape[1], cube.shape[0], 3), dtype=float)
constAll = np.zeros((cube.shape[1], cube.shape[0], 3), dtype=float)
centAll = np.zeros((cube.shape[1], cube.shape[0], 3), dtype=float)
# Loop through spatial pixels and fit lines
for xx in range(cube.shape[0]):
for yy in range(cube.shape[1]):
lineCount = 0.0
for ii in range(len(lines)):
# Get spectrum right around each line
idx = np.where((wavelength > lines[ii]-halfWindow) &
(wavelength < lines[ii]+halfWindow))[0]
spec = cube[xx, yy, idx]
wave = wavelength[idx]
pinit = np.zeros(4, dtype=float)
pinit[0] = spec.max() - spec.min()
pinit[1] = dw * 1.25 / 1000.0
pinit[2] = wave[spec.argmax()]
pinit[3] = spec.min()
out = scipy.optimize.leastsq(fitfunc, pinit,
args=(wave, spec, lines[ii]))
amp, sigma, center, constant = out[0]
# if xx == 2 and yy == 3:
# pdb.set_trace()
if (sigma > 0.0008):
print 'Invalid for xx = %2d, yy = %2d, niter = %d' % \
(xx, yy, out[1])
print ' ', out[0]
continue
lineCount += 1.0
resolutionMap[yy, xx] += lines[ii] / (2.35 * sigma)
ampAll[yy, xx, ii] = amp
resAll[yy, xx, ii] = lines[ii] / (2.35 * sigma)
centAll[yy, xx, ii] = center
constAll[yy, xx, ii] = constant
resolutionMap[yy, xx] /= lineCount
idx1 = np.where(centAll[:,:,0] != 0)
idx2 = np.where(centAll[:,:,1] != 0)
idx3 = np.where(centAll[:,:,2] != 0)
print 'Line Information: '
print ' Center: %8.5f %8.5f %8.5f' % \
(centAll[idx1[0],idx1[1],0].mean(),
centAll[idx2[0],idx2[1],1].mean(),
centAll[idx3[0],idx3[1],2].mean())
print ' : %8.5f %8.5f %8.5f' % \
(centAll[idx1[0],idx1[1],0].std(),
centAll[idx2[0],idx2[1],1].std(),
centAll[idx3[0],idx3[1],2].std())
print ' Amplitudes: %8.2e %8.2e %8.2e' % \
(ampAll[idx1[0],idx1[1],0].mean(),
ampAll[idx2[0],idx2[1],1].mean(),
ampAll[idx3[0],idx3[1],2].mean())
print ' : %8.2e %8.2e %8.2e' % \
(ampAll[idx1[0],idx1[1],0].std(),
ampAll[idx2[0],idx2[1],1].std(),
ampAll[idx3[0],idx3[1],2].std())
print ' Resolution: %8d %8d %8d' % \
(resAll[idx1[0],idx1[1],0].mean(),
resAll[idx2[0],idx2[1],1].mean(),
resAll[idx3[0],idx3[1],2].mean())
print ' : %8d %8d %8d' % \
(resAll[idx1[0],idx1[1],0].std(),
resAll[idx2[0],idx2[1],1].std(),
resAll[idx3[0],idx3[1],2].std())
print ' Constant: %8.2e %8.2e %8.2e' % \
(constAll[idx1[0],idx1[1],0].mean(),
constAll[idx2[0],idx2[1],1].mean(),
constAll[idx3[0],idx3[1],2].mean())
print ' : %8.2e %8.2e %8.2e' % \
(constAll[idx1[0],idx1[1],0].std(),
constAll[idx2[0],idx2[1],1].std(),
constAll[idx3[0],idx3[1],2].std())
py.clf()
py.subplot(1, 2, 1)
py.imshow(cube.sum(axis=2).transpose())
py.title('Flux')
py.subplot(1, 2, 2)
py.imshow(resolutionMap)
py.colorbar()
py.title('Resolution')
py.savefig(workdir + 'plots/map_resolution.png')
# Save the resolution map to a file
mapfile = workdir + 'maps/resolution_map.fits'
ir.imdelete(mapfile)
pyfits.writeto(mapfile, resolutionMap, header=cubehdr)
def makesky(files, nite, wave, skyscale=1):
"""Make short wavelength (not L-band or longer) skies."""
# Start out in something like '06maylgs1/reduce/kp/'
waveDir = os.getcwd() + '/'
redDir = util.trimdir(os.path.abspath(waveDir + '../') + '/')
rootDir = util.trimdir(os.path.abspath(redDir + '../') + '/')
skyDir = waveDir + 'sky_' + nite + '/'
rawDir = rootDir + 'raw/'
util.mkdir(skyDir)
print 'sky dir: ',skyDir
print 'wave dir: ',waveDir
skylist = skyDir + 'skies_to_combine.lis'
output = skyDir + 'sky_' + wave + '.fits'
util.rmall([skylist, output])
nn = [skyDir + 'n' + str(i).zfill(4) for i in files]
nsc = [skyDir + 'scale' + str(i).zfill(4) for i in files]
skies = [rawDir + 'n' + str(i).zfill(4) for i in files]
for ii in range(len(nn)):
ir.imdelete(nn[ii])
ir.imdelete(nsc[ii])
ir.imcopy(skies[ii], nn[ii], verbose="no")
# scale skies to common median
if skyscale:
_skylog = skyDir + 'sky_scale.log'
util.rmall([_skylog])
f_skylog = open(_skylog, 'w')
sky_mean = np.zeros([len(skies)], dtype=float)
for i in range(len(skies)):
text = ir.imstat(nn[i], fields='mean', nclip=4,
lsigma=10, usigma=10, format=0, Stdout=1)
sky_mean[i] = float(text[0])
sky_all = sky_mean.mean()
sky_scale = sky_all/sky_mean
for i in range(len(skies)):
ir.imarith(nn[i], '*', sky_scale[i], nsc[i])
skyf = nn[i].split('/')
print('%s skymean=%10.2f skyscale=%10.2f' %
(skyf[len(skyf)-1], sky_mean[i],sky_scale[i]))
f_skylog.write('%s %10.2f %10.2f\n' %
(nn[i], sky_mean[i], sky_scale[i]))
# Make list for combinng
f_on = open(skylist, 'w')
f_on.write('\n'.join(nsc) + '\n')
f_on.close()
#skylist = skyDir + 'scale????.fits'
f_skylog.close()
else:
# Make list for combinng
f_on = open(skylist, 'w')
f_on.write('\n'.join(nn) + '\n')
f_on.close()
#skylist = skyDir + 'n????.fits'
ir.imdelete(output)
ir.unlearn('imcombine')
ir.imcombine.combine = 'median'
ir.imcombine.reject = 'none'
ir.imcombine.nlow = 1
ir.imcombine.nhigh = 1
ir.imcombine('@' + skylist, output)
#############################
# Change this section if there is PWFS2 shadowing.
# avoid such area
#
# larger possible area "[300:1748,300:1748]" I.e OIWFS
##############################
iraf.niri.niflat.statsec = "[300:1300,300:1500]"
#
#Deleting old flat.fits and f2_bpm.pl and list files fflats.lis,fflatdarks.lis and fshortdarks.lis
#
if createcal=='yes':
iraf.imdelete ("flat.fits,f2_bpm.pl", verify='no')
iraf.delete ("fflats.lis,fflatdarks.lis,fshortdarks.lis", verify='no')
else:
print "Using calibrations already created\n"
print "Using flat.fits"
##
#
# redirect the outpot of the screen to a file!
#
# PLEASE KEEP temp=sys.stdout , and at the end of the screen-to-file, sys.stdout=temp
# this small class does in iraf: sections "*****@*****.**" > "fsky.lis"
#
##
def gmos_img_proc2(dbFile="./raw/obsLog.sqlite3", qd={'use_me': 1,'Instrument': 'GMOS-S', 'CcdBin': '2 2', 'RoI': 'Full', 'Object': 'M8-%', 'DateObs': '2006-09-01:2006-10-30'},
bias_dateobs="2006-09-01:2006-10-30",
biasFlags={'logfile': 'biasLog.txt', 'rawpath': './raw/', 'fl_vardq': 'yes', 'verbose': 'yes'},
flat_dateobs='2006-09-10:2006-10-10',
flatFlags = {'fl_scale': 'yes', 'sctype': 'mean', 'fl_vardq': 'yes','rawpath': './raw/', 'logfile': 'giflatLog.txt', 'verbose': 'yes'},
filters = ['Ha', 'HaC', 'SII', 'r', 'i'],
sciFlags={'fl_over': 'yes', 'fl_trim': 'yes', 'fl_bias':'yes', 'fl_dark': 'no','fl_flat': 'yes', 'logfile':'gireduceLog.txt', 'rawpath': './raw/','fl_vardq': 'yes','bpm':bpm_gmos, 'verbose': 'yes'},
mosaicFlags = {'fl_paste': 'no', 'fl_fixpix': 'no', 'fl_clean': 'yes', 'geointer': 'nearest', 'logfile': 'gmosaicLog.txt', 'fl_vardq': 'yes', 'fl_fulldq': 'yes', 'verbose': 'yes'},
coaddFlags = {'fwhm': 3, 'datamax': 6.e4, 'geointer': 'nearest', 'logfile': 'imcoaddLog.txt'},
targets = ['M8-1', 'M8-2', 'M8-3'],
clean_files = False
):
"""
Parameters
----------
dbFile : str
Filename containing the SQL sqlite3 database created by obslog.py
It must be placed in the ./raw/ directory
Default is `./raw/obsLog.sqlite3`
qd : dictionary
Query Dictionary of essential parameter=value pairs.
Select bias exposures within ~2 months of the target observations
e.g.
qd= {'use_me': 1,
'Instrument': 'GMOS-S', 'CcdBin': '2 2', 'RoI': 'Full', 'Object': 'M8-%',
'DateObs': '2006-09-01:2006-10-30'
}
bias_dateobs : str
String representing the bias search Obsdate
e.g. bias_dateobs = `2006-09-01:2006-10-30`
biasFlags : dict
Dictionary for the keyword flags of gmos.gbias() function
flat_dateobs : str
String representing the flat search Obsdate
e.g. flat_dateobs = `2006-09-10:2006-10-10`
flatFlags : dict
Dictionary for the keyword flags of gmos.giflat() function
e.g. flatFlags = {'fl_scale': 'yes', 'sctype': 'mean', 'fl_vardq': 'yes','rawpath': './raw/',
'logfile': 'giflatLog.txt', 'verbose': 'yes'}
filters : list
List of filter names to perform reduction
e.g. filters=['Ha', 'HaC', 'SII', 'r', 'i']
sciFlags : dict
Dictionary for the keyword flags of gmos.gireduce() function
mosaicFlags : dict
Dictionary for the keyword flags of gmos.gimosaic() function
coaddFlags : dict
Dictionary for the keyword flags of gemtools.imcoadd() function
targets : list
List of names of target observations for the co-addition
e.g. targets = ['M8-1', 'M8-2', 'M8-3']
clean_files : bool
Whether to clean intermediate files from reduction process
Returns
-------
Reduce GMOS imaging based on tutorial example.
"""
print ("### Begin Processing GMOS/MOS Images ###")
print ("###")
print ("=== Creating MasterCals ===")
# From the work_directory:
# Create the query dictionary of essential parameter=value pairs.
# Select bias exposures within ~2 months of the target observations:
print (" --Creating Bias MasterCal--")
qd.update({'DateObs': bias_dateobs})
# Set the task parameters.
gmos.gbias.unlearn()
# The following SQL generates the list of files to process.
SQL = fs.createQuery('bias', qd)
biasFiles = fs.fileListQuery(dbFile, SQL, qd)
# The str.join() function is needed to transform a python list into a string
# filelist that IRAF can understand.
if len(biasFiles) > 1:
files_all = ','.join(str(x) for x in biasFiles)
# import pdb; pdb.set_trace()
gmos.gbias(files_all, 'MCbias.fits', **biasFlags)
# Clean up
year_obs = qd['DateObs'].split('-')[0]
if clean_files:
iraf.imdel('gS{}*.fits'.format(year_obs))
ask_user("MC Bias done. Would you like to continue to proceed with Master Flats? (y/n): ",['y','yes'])
print (" --Creating Twilight Imaging Flat-Field MasterCal--")
# Select flats obtained contemporaneously with the observations.
qd.update({'DateObs': flat_dateobs})
# Set the task parameters.
gmos.giflat.unlearn()
#filters = ['Ha', 'HaC', 'SII', 'r', 'i']
for f in filters:
print " Building twilight flat MasterCal for filter: %s" % (f)
# Select filter name using a substring of the official designation.
qd['Filter2'] = f + '_G%'
mcName = 'MCflat_%s.fits' % (f)
flatFiles = fs.fileListQuery(dbFile, fs.createQuery('twiFlat', qd), qd)
if len(flatFiles) > 0:
files_all = ','.join(str(x) for x in flatFiles)
# import pdb; pdb.set_trace()
gmos.giflat(files_all, mcName, bias='MCbias', **flatFlags)
if clean_files:
iraf.imdel('gS{}*.fits,rgS{}*.fits'.format(year_obs, year_obs))
ask_user("MC Flats done. Would you like to continue to proceed with processing Science Images? (y/n): ", ['yes','y'])
print ("=== Processing Science Images ===")
# Remove restriction on date range
qd['DateObs'] = '*'
prefix = 'rg'
gmos.gireduce.unlearn()
gemtools.gemextn.unlearn() # disarms a bug in gmosaic
gmos.gmosaic.unlearn()
# Reduce the science images, then mosaic the extensions in a loop
for f in filters:
print " Processing science images for filter: %s" % (f)
qd['Filter2'] = f + '_G%'
flatFile = 'MCflat_' + f + '.fits'
SQL = fs.createQuery('sciImg', qd)
sciFiles = fs.fileListQuery(dbFile, SQL, qd)
if len(sciFiles) > 0:
# Make sure BPM table is in sciFlags for employing the imaging Static BPM for this set of detectors.
# import pdb; pdb.set_trace()
all_files = ','.join(str(x) for x in sciFiles)
gmos.gireduce(all_files, bias='MCbias', flat1=flatFile, **sciFlags)
for file in sciFiles:
gmos.gmosaic(prefix + file, **mosaicFlags)
else:
print("No Science images found for filter {}. Check database.".format(f))
import pdb; pdb.set_trace()
if clean_files:
iraf.imdelete('gS{}*.fits,rgS{}*.fits'.format(year_obs,year_obs))
ask_user("Science Images done. Would you like to continue to proceed with image co-addition? (y/n): ", ['y','yes'])
## Co-add the images, per position and filter.
print (" -- Begin image co-addition --")
# Use primarily the default task parameters.
gemtools.imcoadd.unlearn()
prefix = 'mrg'
for f in filters:
print " - Co-addding science images in filter: %s" % (f)
qd['Filter2'] = f + '_G%'
for t in targets:
qd['Object'] = t + '%'
print " - Co-addding science images for position: %s" % (t)
outImage = t + '_' + f + '.fits'
coAddFiles = fs.fileListQuery(dbFile, fs.createQuery('sciImg', qd), qd)
all_files = ','.join(prefix + str(x) for x in coAddFiles)
if all_files == '':
print('No files available for co-addition. Check that the target names are written correctly.')
import pdb; pdb.set_trace()
gemtools.imcoadd(all_files, outimage=outImage, **coaddFlags)
ask_user("Co-addition done. Would you like to clean the latest intermediate reduction files? (y/n): ", ['y','yes'])
if clean_files:
iraf.delete("*_trn*,*_pos,*_cen")
iraf.imdelete("*badpix.pl,*_med.fits,*_mag.fits")
# iraf.imdelete ("mrgS*.fits")
print ("=== Finished Calibration Processing ===")
current_dir = os.getcwd()
os.chdir(ic.dir_iraf)
from pyraf import iraf
from pyraf.iraf import gemini, gmos
os.chdir(current_dir)
iraf.chdir(current_dir)
iraf.unlearn('gfreduce')
iraf.unlearn('gfscatsub')
# ---------- Pre-processing of the science frames ---------- #
# MDF, bias, and overscan
iraf.imdelete('g@' + ic.lst_std)
iraf.imdelete('rg@' + ic.lst_std)
iraf.gfreduce('@' + ic.lst_std,
rawpath=ic.rawdir,
fl_extract='no',
bias=ic.caldir + ic.procbias,
fl_over='yes',
fl_trim='yes',
mdffile=ic.nmdf,
mdfdir='./',
slits=ic.cslit,
line=pk_line,
fl_fluxcal='no',
fl_gscrrej='no',
fl_wavtran='no',
def imalign(images, square=False):
from astropy.wcs import WCS
from astropy.io import fits
img_cat = images[0] # arbitrarily pick the first image to get a catalog with
# fetch a gaia catalog for the full image footprint
outputg = img_cat.f+'.gaia'
gaia_cat = odi.get_gaia_coords(images[0], ota='None', inst='podi', output=outputg)
# print gaia_cat
# convert the ra, dec to image coordinates in each image
# first get the wcs for each image
x0s, y0s, xsizes, ysizes = np.zeros_like(images), np.zeros_like(images), np.zeros_like(images), np.zeros_like(images)
for j,img in enumerate(images):
hdu_img = fits.open(img.f)
img.naxis1 = hdu_img[0].header['NAXIS1']
img.naxis2 = hdu_img[0].header['NAXIS2']
w_img = WCS(hdu_img[0].header)
img.x_img, img.y_img = w_img.all_world2pix(gaia_cat.ra, gaia_cat.dec, 1)
x0s[j], y0s[j] = img.x_img[0], img.y_img[0]
hdu_img.close()
x_ref = np.argmin(x0s)
y_ref = np.argmin(y0s)
# (pick the most positive image as a "reference")
img_ref = images[x_ref]
hdu_ref = fits.open(img_ref.f)
naxis1_ref = hdu_ref[0].header['NAXIS1']
naxis2_ref = hdu_ref[0].header['NAXIS2']
w_ref = WCS(hdu_ref[0].header)
x_ref, y_ref = w_ref.all_world2pix(gaia_cat.ra, gaia_cat.dec, 1)
for j,img in enumerate(images):
# compute the pair-wise integer pixel shifts between the image and the reference
img.x_shift, img.y_shift = np.rint(np.median(x_ref-img.x_img)), np.rint(np.median(y_ref-img.y_img))
img.x_std, img.y_std = np.rint(np.std(x_ref-img.x_img)), np.rint(np.std(y_ref-img.y_img))
# figure out how wide the trimmed image is
img.x_size = np.rint(img.naxis1 + img.x_shift)
img.y_size = np.rint(img.naxis2 + img.y_shift)
# keep track
xsizes[j] = img.x_size
ysizes[j] = img.y_size
# shift the images so that they are aligned to the "reference"
# use relative coordinates-- negative values are applied to the image,
# we will not change the 'reference' because we've already decided it shouldn't shift
iraf.imcopy(img.f, 'temp{:1d}.fits'.format(j))
if img.x_shift < 0.5:
trim_img = 'temp{:1d}.fits[{:d}:{:d},*]'.format(j,int(abs(img.x_shift))+1, img.naxis1)
iraf.imcopy(trim_img, 'temp{:1d}.fits'.format(j))
else:
raise Exception
if img.y_shift < 0.5:
trim_img = 'temp{:1d}.fits[*,{:d}:{:d}]'.format(j,int(abs(img.y_shift))+1, img.naxis2)
iraf.imcopy(trim_img, 'temp{:1d}.fits'.format(j))
else:
raise Exception
# figure out what the smallest image size is
min_xsize = np.min(xsizes)
min_ysize = np.min(ysizes)
for j,img in enumerate(images):
# then take any excess pixels off at the high end of the range in each dimension so that the images have identical dimensions
new_img = img.f[:-5]+'_match.fits'
trim_img = 'temp{:1d}.fits[1:{:d},1:{:d}]'.format(j,int(min_xsize)-1, int(min_ysize)-1)
# make the copy
iraf.imcopy(trim_img, new_img)
# delete the temporary working images
iraf.imdelete('temp{:1d}.fits'.format(j))
#######################################################################
#######################################################################
### Star reduction #
#######################################################################
#######################################################################
#
# Flat reduction
#
for m in range(len(star)):
for i in ['g', 'rg', 'erg']:
iraf.imdelete(i+'@flat'+str(m)+'.list')
# (B) - 'fl_over=no';
iraf.gfreduce('@flat'+str(m)+'.list', slits='header',
rawpath='rawdir$', fl_inter='no',
fl_addmdf='yes', key_mdf='MDF', mdffile='default', weights='no',
fl_over='no', fl_trim='yes', fl_bias='yes', trace='yes', t_order=4,
fl_flux='no', fl_gscrrej='no', fl_extract='yes', fl_gsappwave='no',
fl_wavtran='no', fl_novl='no', fl_skysub='no', reference='',
recenter='yes', fl_vardq='yes', bias='caldir$'+bias[m][0])
#
# Twilight reduction
#
for m in range(len(star)):
def mscimage(input=None, output=None, format='image', pixmask=no,verbose=')_.verbose',wcssource='image',reference='',ra=INDEF,dec=INDEF,scale=INDEF,rotation=INDEF,blank=0.0,interpolant='poly5',minterpolant='linear',boundary='reflect',constant=0.0,fluxconserve=no,ntrim=8,nxblock=INDEF,nyblock=INDEF,interactive=no,nx=10,ny=20,fitgeometry='general',xxorder=4,xyorder=4,xxterms='half',yxorder=4,yyorder=4,yxterms='half',fd_in='',fd_ext='',fd_coord='',mode='ql',DOLLARnargs=0,taskObj=None):
Vars = IrafParList('mscimage')
Vars.addParam(makeIrafPar(input, datatype='string', name='input', mode='a',prompt='List of input mosaic exposures'))
Vars.addParam(makeIrafPar(output, datatype='string', name='output',mode='a',prompt='List of output images'))
Vars.addParam(makeIrafPar(format, datatype='string', name='format',enum=['image', 'mef'],mode='h',prompt='Output format (image|mef)'))
Vars.addParam(makeIrafPar(pixmask, datatype='bool', name='pixmask',mode='h',prompt='Create pixel mask?'))
Vars.addParam(makeIrafPar(verbose, datatype='bool', name='verbose',mode='h',prompt='Verbose output?\n\n# Output WCS parameters'))
Vars.addParam(makeIrafPar(wcssource, datatype='string', name='wcssource',enum=['image', 'parameters', 'match'],mode='h',prompt='Output WCS source (image|parameters|match)'))
Vars.addParam(makeIrafPar(reference, datatype='file', name='reference',mode='h',prompt='Reference image'))
Vars.addParam(makeIrafPar(ra, datatype='real', name='ra', max=24.0,min=0.0,mode='h',prompt='RA of tangent point (hours)'))
Vars.addParam(makeIrafPar(dec, datatype='real', name='dec', max=90.0,min=-90.0,mode='h',prompt='DEC of tangent point (degrees)'))
Vars.addParam(makeIrafPar(scale, datatype='real', name='scale', mode='h',prompt='Scale (arcsec/pixel)'))
Vars.addParam(makeIrafPar(rotation, datatype='real', name='rotation',max=360.0,min=-360.0,mode='h',prompt='Rotation of DEC from N to E (degrees)\n\n# Resampling parmeters'))
Vars.addParam(makeIrafPar(blank, datatype='real', name='blank', mode='h',prompt='Blank value'))
Vars.addParam(makeIrafPar(interpolant, datatype='string',name='interpolant',mode='h',prompt='Interpolant for data'))
Vars.addParam(makeIrafPar(minterpolant, datatype='string',name='minterpolant',mode='h',prompt='Interpolant for mask'))
Vars.addParam(makeIrafPar(boundary, datatype='string', name='boundary',enum=['nearest', 'constant', 'reflect', 'wrap'],mode='h',prompt='Boundary extension'))
Vars.addParam(makeIrafPar(constant, datatype='real', name='constant',mode='h',prompt='Constant boundary extension value'))
Vars.addParam(makeIrafPar(fluxconserve, datatype='bool',name='fluxconserve',mode='h',prompt='Preserve flux per unit area?'))
Vars.addParam(makeIrafPar(ntrim, datatype='int', name='ntrim', min=0,mode='h',prompt='Edge trim in each extension'))
Vars.addParam(makeIrafPar(nxblock, datatype='int', name='nxblock',mode='h',prompt='X dimension of working block size in pixels'))
Vars.addParam(makeIrafPar(nyblock, datatype='int', name='nyblock',mode='h',prompt='Y dimension of working block size in pixels\n\n# Geometric mapping parameters'))
Vars.addParam(makeIrafPar(interactive, datatype='bool', name='interactive',mode='h',prompt='Fit mapping interactively?'))
Vars.addParam(makeIrafPar(nx, datatype='int', name='nx', mode='h',prompt='Number of x grid points'))
Vars.addParam(makeIrafPar(ny, datatype='int', name='ny', mode='h',prompt='Number of y grid points'))
Vars.addParam(makeIrafPar(fitgeometry, datatype='string',name='fitgeometry',enum=['shift', 'xyscale', 'rotate', 'rscale', 'rxyscale', 'general'],mode='h',prompt='Fitting geometry'))
Vars.addParam(makeIrafPar(xxorder, datatype='int', name='xxorder', min=2,mode='h',prompt='Order of x fit in x'))
Vars.addParam(makeIrafPar(xyorder, datatype='int', name='xyorder', min=2,mode='h',prompt='Order of x fit in y'))
Vars.addParam(makeIrafPar(xxterms, datatype='string', name='xxterms',mode='h',prompt='X fit cross terms type'))
Vars.addParam(makeIrafPar(yxorder, datatype='int', name='yxorder', min=2,mode='h',prompt='Order of y fit in x'))
Vars.addParam(makeIrafPar(yyorder, datatype='int', name='yyorder', min=2,mode='h',prompt='Order of y fit in y'))
Vars.addParam(makeIrafPar(yxterms, datatype='string', name='yxterms',mode='h',prompt='Y fit cross terms type\n\n'))
Vars.addParam(makeIrafPar(fd_in, datatype='struct', name='fd_in',list_flag=1,mode='h',prompt=''))
Vars.addParam(makeIrafPar(fd_ext, datatype='struct', name='fd_ext',list_flag=1,mode='h',prompt=''))
Vars.addParam(makeIrafPar(fd_coord, datatype='struct', name='fd_coord',list_flag=1,mode='h',prompt=''))
Vars.addParam(makeIrafPar(mode, datatype='string', name='mode', mode='h',prompt=''))
Vars.addParam(makeIrafPar(DOLLARnargs, datatype='int', name='$nargs',mode='h'))
Vars.addParam(makeIrafPar(None, datatype='file', name='in', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='file', name='out', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='file', name='ref', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='file', name='pl', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='file', name='image', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='file', name='trimsec', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='file', name='outsec', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='file', name='plsec', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='file', name='inlists', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='file', name='extlist', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='file', name='pllist', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='file', name='coord', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='file', name='db', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='file', name='wcsref', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='file', name='outtemp', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='file', name='pltemp', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='int', name='nc', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='int', name='nl', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='int', name='ncref', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='int', name='nlref', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='int', name='cmin', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='int', name='cmax', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='int', name='lmin', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='int', name='lmax', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='int', name='nimage', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='int', name='nimages', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='int', name='nxblk', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='int', name='nyblk', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='real', name='x', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='real', name='y', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='real', name='rval', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='real', name='xmin', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='real', name='xmax', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='real', name='ymin', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='real', name='ymax', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='real', name='crpix1', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='real', name='crpix2', mode='u'))
Vars.addParam(makeIrafPar(None, datatype='string', name='extname',mode='u'))
Vars.addParam(makeIrafPar(None, datatype='string', name='str', mode='u'))
iraf.cache('mscextensions', 'mscgmask')
Vars.inlists = iraf.mktemp('tmp$iraf')
Vars.extlist = iraf.mktemp('tmp$iraf')
Vars.pllist = iraf.mktemp('tmp$iraf')
Vars.coord = iraf.mktemp('tmp$iraf')
Vars.db = iraf.mktemp('tmp$iraf')
Vars.outtemp = iraf.mktemp('tmp')
Vars.wcsref = iraf.mktemp('tmp')
Vars.pltemp = iraf.mktemp('tmp')
iraf.joinlists(Vars.input, Vars.output, output = Vars.inlists, delim = ' ',short=yes,type = 'image')
Vars.fd_in = Vars.inlists
while (iraf.fscan(locals(), 'Vars.fd_in', 'Vars.PYin', 'Vars.out') != EOF):
if (iraf.imaccess(Vars.out)):
iraf.printf('Warning: Image already exists (%s)\n', Vars.out)
continue
if (Vars.pixmask):
Vars.pl = Vars.out
Vars.nc = iraf.strlen(Vars.pl)
if (Vars.nc > 5 and iraf.substr(Vars.pl, Vars.nc - 4, Vars.nc) == '.fits'):
Vars.pl = iraf.substr(Vars.pl, 1, Vars.nc - 5)
elif (Vars.nc > 4 and iraf.substr(Vars.out, Vars.nc - 3, Vars.nc) == '.imh'):
Vars.pl = iraf.substr(Vars.pl, 1, Vars.nc - 4)
Vars.pl = Vars.pl + '_bpm'
if (Vars.format == 'image' and iraf.imaccess(Vars.pl)):
iraf.printf('Warning: Mask already exists (%s)\n', Vars.pl)
continue
else:
Vars.pl = ''
iraf.mscextensions(Vars.PYin, output = 'file', index = '0-',extname = '',extver = '',lindex = no,lname = yes,lver = no,ikparams = '',Stdout=Vars.extlist)
Vars.nimages = int(iraf.mscextensions.nimages)
Vars.nimage = 0
if (Vars.nimages < 1):
iraf.printf("WARNING: No input image data found in `%s'.\
",Vars.PYin)
iraf.delete(Vars.extlist, verify = no)
continue
if (not iraf.imaccess(Vars.wcsref)):
Vars.ref = Vars.reference
if (Vars.wcssource == 'match'):
Vars.wcsref = Vars.ref
else:
iraf.mscwtemplate('@' + Vars.extlist, Vars.wcsref,wcssource = Vars.wcssource,reference = Vars.ref,ra = Vars.ra,dec = Vars.dec,scale = Vars.scale,rotation = Vars.rotation,projection = '',verbose = Vars.verbose)
Vars.fd_ext = Vars.extlist
while (iraf.fscan(locals(), 'Vars.fd_ext', 'Vars.image') != EOF):
Vars.nimage = Vars.nimage + 1
if (Vars.nimages > 1):
Pipe1 = iraf.hselect(Vars.image, 'extname', yes, Stdout=1)
iraf.scan(locals(), 'Vars.extname', Stdin=Pipe1)
del Pipe1
if (iraf.nscan() == 0):
Vars.extname = 'im' + str(Vars.nimage)
Pipe1 = iraf.printf('%s[%s,append]\n', Vars.outtemp,Vars.extname,Stdout=1)
iraf.scan(locals(), 'Vars.outsec', Stdin=Pipe1)
del Pipe1
Pipe1 = iraf.printf('%s%s\n', Vars.pl, Vars.extname, Stdout=1)
iraf.scan(locals(), 'Vars.plsec', Stdin=Pipe1)
del Pipe1
else:
Vars.extname = ''
Vars.outsec = Vars.outtemp
Vars.plsec = Vars.pl
if (Vars.pixmask and iraf.imaccess(Vars.plsec)):
iraf.delete(Vars.coord, verify = no)
iraf.delete(Vars.db, verify = no)
iraf.printf('Warning: Mask already exists (%s)\n', Vars.plsec)
continue
if (Vars.verbose):
iraf.printf('Resampling %s ...\n', Vars.image)
Pipe1 = iraf.hselect(Vars.image, 'naxis1,naxis2', yes, Stdout=1)
iraf.scan(locals(), 'Vars.nc', 'Vars.nl', Stdin=Pipe1)
del Pipe1
Vars.cmin = 1 + Vars.ntrim
Vars.cmax = Vars.nc - Vars.ntrim
Vars.lmin = 1 + Vars.ntrim
Vars.lmax = Vars.nl - Vars.ntrim
Pipe1 = iraf.printf('[%d:%d,%d:%d]\n', Vars.cmin, Vars.cmax,Vars.lmin,Vars.lmax,Stdout=1)
iraf.scan(locals(), 'Vars.trimsec', Stdin=Pipe1)
del Pipe1
if (Vars.wcssource == 'match'):
Pipe1 = iraf.hselect(Vars.ref, 'naxis1,naxis2', yes, Stdout=1)
iraf.scan(locals(), 'Vars.ncref', 'Vars.nlref', Stdin=Pipe1)
del Pipe1
Vars.xmin = (Vars.ncref - 1.) / (Vars.nx - 1.)
Vars.ymin = (Vars.nlref - 1.) / (Vars.ny - 1.)
Vars.ymax = 1
while (Vars.ymax <= Vars.nlref + 1):
Vars.xmax = 1
while (Vars.xmax <= Vars.ncref + 1):
iraf.clPrint(Vars.xmax, Vars.ymax, Vars.xmax,Vars.ymax,StdoutAppend=Vars.coord)
Vars.xmax = Vars.xmax + Vars.xmin
Vars.ymax = Vars.ymax + Vars.ymin
iraf.mscctran(Vars.coord, Vars.db, Vars.ref, 'logical','world',columns = '3 4',units = '',formats = '%.4H %.3h',min_sigdigit = 10,verbose = no)
iraf.delete(Vars.coord, verify=no)
iraf.wcsctran(Vars.db, Vars.coord, Vars.image + Vars.trimsec,inwcs = 'world',outwcs = 'logical',columns = '3 4',units = 'hours native',formats = '',min_sigdigit = 10,verbose = no)
iraf.delete(Vars.db, verify=no)
else:
Vars.nc = Vars.cmax - Vars.cmin + 1
Vars.nl = Vars.lmax - Vars.lmin + 1
Vars.xmin = (Vars.nc - 1.) / (Vars.nx - 1.)
Vars.ymin = (Vars.nl - 1.) / (Vars.ny - 1.)
Vars.ymax = 1
while (Vars.ymax <= Vars.nl + 1):
Vars.xmax = 1
while (Vars.xmax <= Vars.nc + 1):
iraf.clPrint(Vars.xmax, Vars.ymax, Vars.xmax,Vars.ymax,StdoutAppend=Vars.coord)
Vars.xmax = Vars.xmax + Vars.xmin
Vars.ymax = Vars.ymax + Vars.ymin
iraf.mscctran(Vars.coord, Vars.db, Vars.image + Vars.trimsec,'logical','world',columns = '1 2',units = '',formats = '%.4H %.3h',min_sigdigit = 10,verbose = no)
iraf.delete(Vars.coord, verify=no)
iraf.wcsctran(Vars.db, Vars.coord, Vars.wcsref,inwcs = 'world',outwcs = 'logical',columns = '1 2',units = 'hours native',formats = '',min_sigdigit = 10,verbose = no)
iraf.delete(Vars.db, verify=no)
Vars.xmax = 0.
Vars.xmin = 1.
Vars.ymax = 0.
Vars.ymin = 1.
Vars.fd_coord = Vars.coord
while (iraf.fscan(locals(), 'Vars.fd_coord', 'Vars.x', 'Vars.y') != EOF):
if (iraf.nscan() < 2):
continue
if (Vars.xmax < Vars.xmin):
Vars.xmin = Vars.x
Vars.xmax = Vars.x
Vars.ymin = Vars.y
Vars.ymax = Vars.y
else:
Vars.xmin = float(iraf.minimum(Vars.x, Vars.xmin))
Vars.xmax = float(iraf.maximum(Vars.x, Vars.xmax))
Vars.ymin = float(iraf.minimum(Vars.y, Vars.ymin))
Vars.ymax = float(iraf.maximum(Vars.y, Vars.ymax))
Vars.fd_coord = ''
if (Vars.xmax <= Vars.xmin or Vars.ymax <= Vars.ymin):
iraf.error(1, 'No overlap for matching reference')
Vars.cmin = int(iraf.nint(Vars.xmin - 1.5))
Vars.cmax = int(iraf.nint(Vars.xmax + 1.5))
Vars.lmin = int(iraf.nint(Vars.ymin - 1.5))
Vars.lmax = int(iraf.nint(Vars.ymax + 1.5))
iraf.geomap(Vars.coord, Vars.db, Vars.cmin, Vars.cmax, Vars.lmin,Vars.lmax,transforms = '',results = '',fitgeometry = Vars.fitgeometry,function = 'chebyshev',xxorder = Vars.xxorder,xyorder = Vars.xyorder,xxterms = Vars.xxterms,yxorder = Vars.yxorder,yyorder = Vars.yyorder,yxterms = Vars.yxterms,reject = INDEF,calctype = 'double',verbose = no,interactive = Vars.interactive,graphics = 'stdgraph',cursor = '')
if (Vars.wcssource == 'match'):
Vars.cmin = 1
Vars.lmin = 1
Vars.cmax = Vars.ncref
Vars.lmax = Vars.nlref
if (Vars.nxblock == INDEF):
Vars.nxblk = Vars.cmax - Vars.cmin + 3
else:
Vars.nxblk = Vars.nxblock
if (Vars.nyblock == INDEF):
Vars.nyblk = Vars.lmax - Vars.lmin + 3
else:
Vars.nyblk = Vars.nyblock
iraf.geotran(Vars.image + Vars.trimsec, Vars.outsec, Vars.db,Vars.coord,geometry = 'geometric',xin = INDEF,yin = INDEF,xshift = INDEF,yshift = INDEF,xout = INDEF,yout = INDEF,xmag = INDEF,ymag = INDEF,xrotation = INDEF,yrotation = INDEF,xmin = Vars.cmin,xmax = Vars.cmax,ymin = Vars.lmin,ymax = Vars.lmax,xsample = 10.,ysample = 10.,xscale = 1.,yscale = 1.,ncols = INDEF,nlines = INDEF,interpolant = Vars.interpolant,boundary = 'constant',constant = Vars.constant,fluxconserve = Vars.fluxconserve,nxblock = Vars.nxblk,nyblock = Vars.nyblk,verbose = no)
iraf.wcscopy(Vars.outsec, Vars.wcsref, verbose=no)
Vars.xmin = 0.
Vars.ymin = 0.
Pipe1 = iraf.hselect(Vars.outsec, 'crpix1,crpix2', yes, Stdout=1)
iraf.scan(locals(), 'Vars.xmin', 'Vars.ymin', Stdin=Pipe1)
del Pipe1
Vars.xmin = Vars.xmin - Vars.cmin + 1
Vars.ymin = Vars.ymin - Vars.lmin + 1
if (Vars.nimage == 1):
Vars.crpix1 = Vars.xmin
Vars.crpix2 = Vars.ymin
else:
Vars.crpix1 = float(iraf.maximum(Vars.crpix1, Vars.xmin))
Vars.crpix2 = float(iraf.maximum(Vars.crpix2, Vars.ymin))
iraf.hedit(Vars.outsec, 'crpix1', Vars.xmin, add=yes, verify=no,show=no,update=yes)
iraf.hedit(Vars.outsec, 'crpix2', Vars.ymin, add=yes, verify=no,show=no,update=yes)
if (Vars.pixmask):
Pipe1 = iraf.printf('%s%s\n', Vars.pl, Vars.extname, Stdout=1)
iraf.scan(locals(), 'Vars.plsec', Stdin=Pipe1)
del Pipe1
iraf.mscgmask(Vars.image + Vars.trimsec, Vars.pltemp + '.pl','BPM',mval = 10000)
iraf.geotran(Vars.pltemp, Vars.plsec + '.fits', Vars.db,Vars.coord,geometry = 'geometric',xin = INDEF,yin = INDEF,xshift = INDEF,yshift = INDEF,xout = INDEF,yout = INDEF,xmag = INDEF,ymag = INDEF,xrotation = INDEF,yrotation = INDEF,xmin = Vars.cmin,xmax = Vars.cmax,ymin = Vars.lmin,ymax = Vars.lmax,xsample = 10.,ysample = 10.,interpolant = Vars.minterpolant,boundary = 'constant',constant = 20000.,fluxconserve = no,nxblock = Vars.nxblk,nyblock = Vars.nyblk,verbose = no)
iraf.imdelete(Vars.pltemp, verify=no)
iraf.mscpmask(Vars.plsec + '.fits', Vars.plsec + '.pl')
iraf.imdelete(Vars.plsec + '.fits', verify=no)
iraf.hedit(Vars.outsec, 'BPM', Vars.plsec + '.pl', add=yes,show=no,verify=no,update=yes)
iraf.wcscopy(Vars.plsec, Vars.outsec, verbose=no)
iraf.clPrint(Vars.plsec, StdoutAppend=Vars.pllist)
else:
iraf.hedit(Vars.outsec, 'BPM', PYdel=yes, add=no, addonly=no,show=no,verify=no,update=yes)
iraf.delete(Vars.coord, verify = no)
iraf.delete(Vars.db, verify = no)
Vars.fd_ext = ''
iraf.delete(Vars.extlist, verify = no)
if (Vars.nimages > 1 and Vars.format == 'image'):
if (Vars.verbose):
iraf.printf('Creating image %s ...\n', Vars.out)
iraf.mscextensions(Vars.outtemp, output = 'file', index = '',extname = '',extver = '',lindex = no,lname = yes,lver = no,ikparams = '',Stdout=Vars.extlist)
if (Vars.pixmask):
iraf.combine('@' + Vars.pllist, Vars.pltemp + '.pl',headers = '',bpmasks = Vars.pl,rejmasks = '',nrejmasks = '',expmasks = '',sigmas = '',imcmb = '',ccdtype = '',amps = no,subsets = no,delete = no,combine = 'average',reject = 'none',project = no,outtype = 'real',outlimits = '',offsets = 'wcs',masktype = 'none',maskvalue = '0',blank = 0.,scale = 'none',zero = 'none',weight = 'none',statsec = '',lthreshold = INDEF,hthreshold = 0.99,nlow = 1,nhigh = 1,nkeep = 1,mclip = yes,lsigma = 3.,hsigma = 3.,rdnoise = '0.',gain = '1.',snoise = '0.',sigscale = 0.1,pclip = - 0.5,grow = 0.,Stdout='dev$null')
iraf.imdelete(Vars.pltemp, verify=no)
iraf.combine('@' + Vars.extlist, Vars.out, headers = '',bpmasks = '',rejmasks = '',nrejmasks = '',expmasks = '',sigmas = '',imcmb = '',ccdtype = '',amps = no,subsets = no,delete = no,combine = 'average',reject = 'none',project = no,outtype = 'real',outlimits = '',offsets = 'wcs',masktype = 'badvalue',maskvalue = '2',blank = 0.,scale = 'none',zero = 'none',weight = 'none',statsec = '',lthreshold = INDEF,hthreshold = INDEF,nlow = 1,nhigh = 1,nkeep = 1,mclip = yes,lsigma = 3.,hsigma = 3.,rdnoise = '0.',gain = '1.',snoise = '0.',sigscale = 0.1,pclip = - 0.5,grow = 0.,Stdout='dev$null')
iraf.hedit(Vars.out, 'BPM', Vars.pl, add=yes, verify=no,show=no,update=yes)
iraf.hedit(Vars.pl, 'IMCMB???,PROCID??', add=no, addonly=no,PYdel=yes,update=yes,verify=no,show=no)
else:
iraf.combine('@' + Vars.extlist, Vars.out, headers = '',bpmasks = '',rejmasks = '',nrejmasks = '',expmasks = '',sigmas = '',imcmb = '',ccdtype = '',amps = no,subsets = no,delete = no,combine = 'average',reject = 'none',project = no,outtype = 'real',outlimits = '',offsets = 'wcs',masktype = 'none',maskvalue = '2',blank = 0.,scale = 'none',zero = 'none',weight = 'none',statsec = '',lthreshold = INDEF,hthreshold = INDEF,nlow = 1,nhigh = 1,nkeep = 1,mclip = yes,lsigma = 3.,hsigma = 3.,rdnoise = '0.',gain = '1.',snoise = '0.',sigscale = 0.1,pclip = - 0.5,grow = 0.,Stdout='dev$null')
Pipe2 = iraf.hselect('@' + Vars.extlist, 'gain', yes, Stdout=1)
Pipe1 = iraf.average(data_value = 0., Stdin=Pipe2, Stdout=1)
del Pipe2
iraf.scan(locals(), 'Vars.rval', Stdin=Pipe1)
del Pipe1
iraf.hedit(Vars.out, 'gain', Vars.rval, add=yes, PYdel=no,update=yes,verify=no,show=no)
Pipe2 = iraf.hselect('@' + Vars.extlist, 'rdnoise', yes, Stdout=1)
Pipe1 = iraf.average(data_value = 0., Stdin=Pipe2, Stdout=1)
del Pipe2
iraf.scan(locals(), 'Vars.rval', Stdin=Pipe1)
del Pipe1
iraf.hedit(Vars.out, 'rdnoise', Vars.rval, add=yes, PYdel=no,update=yes,verify=no,show=no)
iraf.hedit(Vars.out, 'IMCMB???,PROCID??', add=no, addonly=no,PYdel=yes,update=yes,verify=no,show=no)
iraf.hedit(Vars.out,'NEXTEND,DETSEC,CCDSEC,AMPSEC,IMAGEID,DATASEC,TRIMSEC,BIASSEC',add=no,addonly=no,PYdel=yes,update=yes,verify=no,show=no)
iraf.imdelete(Vars.outtemp, verify=no)
if (iraf.access(Vars.pllist)):
iraf.imdelete('@' + Vars.pllist, verify=no)
iraf.delete(Vars.pllist, verify=no)
iraf.delete(Vars.extlist, verify = no)
elif (Vars.nimages > 1):
iraf.imrename(Vars.outtemp, Vars.out, verbose=no)
iraf.mscextensions(Vars.out, output = 'file', index = '',extname = '',extver = '',lindex = no,lname = yes,lver = no,ikparams = '',Stdout=Vars.extlist)
Vars.fd_ext = Vars.extlist
while (iraf.fscan(locals(), 'Vars.fd_ext', 'Vars.image') != EOF):
Pipe1 = iraf.hselect(Vars.image, 'naxis1,naxis2,crpix1,crpix2',yes,Stdout=1)
iraf.scan(locals(), 'Vars.nc', 'Vars.nl', 'Vars.xmin','Vars.ymin',Stdin=Pipe1)
del Pipe1
Vars.cmin = int(iraf.nint(Vars.crpix1 - Vars.xmin + 1))
Vars.lmin = int(iraf.nint(Vars.crpix2 - Vars.ymin + 1))
Vars.cmax = Vars.nc + Vars.cmin - 1
Vars.lmax = Vars.nl + Vars.lmin - 1
Pipe1 = iraf.printf('[%d:%d,%d:%d]\n', Vars.cmin, Vars.cmax,Vars.lmin,Vars.lmax,Stdout=1)
iraf.scan(locals(), 'Vars.str', Stdin=Pipe1)
del Pipe1
iraf.hedit(Vars.image, 'DETSEC', Vars.str, add=yes, verify=no,show=no,update=yes)
iraf.hedit(Vars.image, 'DTM1_1', 1., add=yes, verify=no,show=no,update=yes)
iraf.hedit(Vars.image, 'DTM2_2', 1., add=yes, verify=no,show=no,update=yes)
Vars.cmin = Vars.cmin - 1
Vars.lmin = Vars.lmin - 1
iraf.hedit(Vars.image, 'DTV1', Vars.cmin, add=yes, verify=no,show=no,update=yes)
iraf.hedit(Vars.image, 'DTV2', Vars.lmin, add=yes, verify=no,show=no,update=yes)
iraf.hedit(Vars.image,'CCDSUM,CCDSEC,AMPSEC,ATM1_1,ATM2_2,ATV1,ATV2',PYdel=yes,add=no,addonly=no,verify=no,show=no,update=yes)
Vars.fd_ext = ''
iraf.delete(Vars.extlist, verify=no)
else:
iraf.imrename(Vars.outsec, Vars.out, verbose=no)
if (iraf.access(Vars.pllist)):
iraf.delete(Vars.pllist, verify=no)
Vars.fd_in = ''
iraf.delete(Vars.inlists, verify = no)
if (Vars.wcssource != 'match' and iraf.imaccess(Vars.wcsref)):
iraf.imdelete(Vars.wcsref, verify=no)
def reduce_stdstar(rawdir, rundir, caldir, starobj, stdstar, flat,
arc, twilight, starimg, bias, overscan, vardq):
"""
Reduction pipeline for standard star.
Parameters
----------
rawdir: string
Directory containing raw images.
rundi: string
Directory where processed files are saved.
caldir: string
Directory containing standard star calibration files.
starobj: string
Object keyword for the star image.
stdstar: string
Star name in calibration file.
flat: list
Names of the files containing flat field images.
arc: list
Arc images.
twilight: list
Twilight flat images.
starimg: string
Name of the file containing the image to be reduced.
bias: list
Bias images.
"""
iraf.set(stdimage='imtgmos')
iraf.gemini()
iraf.gemtools()
iraf.gmos()
#iraf.unlearn('gemini')
#iraf.unlearn('gmos')
iraf.task(lacos_spec='/storage/work/gemini_pairs/lacos_spec.cl')
tstart = time.time()
#set directories
iraf.set(caldir=rawdir) #
iraf.set(rawdir=rawdir) # raw files
iraf.set(procdir=rundir) # processed files
iraf.gmos.logfile='logfile.log'
iraf.cd('procdir')
# building lists
def range_string(l):
return (len(l)*'{:4s},').format(*[i[-9:-5] for i in l])
iraf.gemlist(range=range_string(flat), root=flat[0][:-9],
Stdout='flat.list')
iraf.gemlist(range=range_string(arc), root=arc[0][:-9],
Stdout='arc.list')
#iraf.gemlist(range=range_string(star), root=star[0][:-4],
# Stdout='star.list')
iraf.gemlist(range=range_string(twilight),
root=twilight[0][:-9], Stdout='twilight.list')
iraf.gfreduce.bias = 'caldir$'+bias[0]
#######################################################################
#######################################################################
### Star reduction #
#######################################################################
#######################################################################
#
# Flat reduction
#
iraf.gfreduce(
'@flat.list', slits='header', rawpath='rawdir$', fl_inter='no',
fl_addmdf='yes', key_mdf='MDF', mdffile='default', weights='no',
fl_over=overscan, fl_trim='yes', fl_bias='yes', trace='yes', t_order=4,
fl_flux='no', fl_gscrrej='no', fl_extract='yes', fl_gsappwave='no',
fl_wavtran='no', fl_novl='no', fl_skysub='no', reference='',
recenter='yes', fl_vardq=vardq)
iraf.gfreduce('@twilight.list', slits='header', rawpath='rawdir$',
fl_inter='no', fl_addmdf='yes', key_mdf='MDF',
mdffile='default', weights='no',
fl_over=overscan, fl_trim='yes', fl_bias='yes', trace='yes',
recenter='no',
fl_flux='no', fl_gscrrej='no', fl_extract='yes', fl_gsappwave='no',
fl_wavtran='no', fl_novl='no', fl_skysub='no',
reference='erg'+flat[0], fl_vardq=vardq)
#
# Response function
#
for i, j in enumerate(flat):
j = j[:-5]
iraf.imdelete(j+'_response')
iraf.gfresponse('erg'+j+'.fits', out='erg'+j+'_response',
skyimage='erg'+twilight[i], order=95, fl_inter='no',
func='spline3',
sample='*', verbose='yes')
# Arc reduction
#
iraf.gfreduce(
'@arc.list', slits='header', rawpath='rawdir$', fl_inter='no',
fl_addmdf='yes', key_mdf='MDF', mdffile='default', weights='no',
fl_over=overscan, fl_trim='yes', fl_bias='yes', trace='no',
recenter='no',
fl_flux='no', fl_gscrrej='no', fl_extract='yes', fl_gsappwave='no',
fl_wavtran='no', fl_novl='no', fl_skysub='no', reference='erg'+flat[0],
fl_vardq=vardq)
# Finding wavelength solution
# Note: the automatic identification is very good
#
for i in arc:
iraf.gswavelength('erg'+i, function='chebyshev', nsum=15, order=4,
fl_inter='no', nlost=5, ntarget=20, aiddebug='s', threshold=5,
section='middle line')
#
# Apply wavelength solution to the lamp 2D spectra
#
iraf.gftransform('erg'+i, wavtran='erg'+i, outpref='t', fl_vardq=vardq)
##
## Actually reduce star
##
iraf.gfreduce(
starimg, slits='header', rawpath='rawdir$', fl_inter='no',
fl_addmdf='yes', key_mdf='MDF', mdffile='default', weights='no',
fl_over=overscan, fl_trim='yes', fl_bias='yes', trace='no',
recenter='no',
fl_flux='no', fl_gscrrej='no', fl_extract='no', fl_gsappwave='no',
fl_wavtran='no', fl_novl='yes', fl_skysub='no', fl_vardq=vardq)
iraf.gemcrspec('rg{:s}'.format(starimg), out='lrg'+starimg, sigfrac=0.32,
niter=4, fl_vardq=vardq)
iraf.gfreduce(
'lrg'+starimg, slits='header', rawpath='./', fl_inter='no',
fl_addmdf='no', key_mdf='MDF', mdffile='default',
fl_over='no', fl_trim='no', fl_bias='no', trace='no',
recenter='no',
fl_flux='no', fl_gscrrej='no', fl_extract='yes',
fl_gsappwave='yes',
fl_wavtran='yes', fl_novl='no', fl_skysub='yes',
reference='erg'+flat[0][:-5], weights='no',
wavtraname='erg'+arc[0][:-5],
response='erg'+flat[0][:-5]+'_response.fits',
fl_vardq=vardq)
#
# Apsumming the stellar spectra
#
iraf.gfapsum(
'stexlrg'+starimg, fl_inter='no', lthreshold=400.,
reject='avsigclip')
#
# Building sensibility function
#
iraf.gsstandard(
('astexlrg{:s}').format(starimg), starname=stdstar,
observatory='Gemini-South', sfile='std', sfunction='sens',
caldir=caldir)
#
# Apply flux calibration to galaxy
#
#
##iraf.imdelete('*****@*****.**')
#
##iraf.gscalibrate('*****@*****.**',sfunction='sens.fits',fl_ext='yes',extinct='onedstds$ctioextinct.dat',observatory='Gemini-South',fluxsca=1)
#
##
## Create data cubes
##
#
#
##for i in objs:
## iraf.imdelete('d0.1cstexlrg'+i+'.fits')
## iraf.gfcube('cstexlrg'+i+'.fits',outpref='d0.1',ssample=0.1,fl_atmd='yes',fl_flux='yes')
#
##
## Combine cubes
##
#
#
##iraf.imdelete('am2306-721r4_wcsoffsets.fits')
##iraf.imcombine('d0.1cstexlrgS20141113S00??.fits[1]',output='am2306-721r4_wcsoffsets.fits',combine='average',reject='sigclip',masktype='badvalue',lsigma=2,hsigma=2,offset='wcs',outlimits='2 67 2 48 100 1795')
#
tend = time.time()
print('Elapsed time in reduction: {:.2f}'.format(tend - tstart))
def maskStars(imageName):
#read in the fits data and header
imageHeader = fits.open(imageName+".fits")[0]
#Read in the skysigma, skyv, and seeing (fwhm) from the header and set up other relevant values
sigma = imageHeader.header['SKYSIGMA']
threshold = 3.0 * sigma
skyv = imageHeader.header['SKY']
fwhm = imageHeader.header['SEEING']
aper = 3.0 * fwhm
annul = 5 * fwhm
#Read in the galcut file
galcut = open("galcut.file")
galcutString = file.read(galcut)
galcut.close()
#Parse the galcut file into its variables
galList = galcutString.split(' ')
xCenter = float(galList[0])
yCenter = float(galList[1])
a = float(galList[2]) #Semimajor Axis
eccent = float(galList[3]) #Eccentricity
posAngle = float(galList[4]) #Position angle
posAngleRad = (posAngle+90)*3.14159/180
b = a * eccent #Semiminor Axis
na = -1*a
nb = -1*b
#Create a version of the r image with the sky added back in for the daofind procedure
iraf.imdelete("withsky")
iraf.imarith(imageName,'+',skyv,"withsky")
# Load daophot package
iraf.digiphot()
iraf.daophot()
print("A. Find stars using daofind")
#DAOFIND searches the IRAF images image for local density maxima,
# with a full-width half-maxima of datapars.fwhmpsf, and a peak
# amplitude greater than findpars.threshold * datapars.sigma above
# the local background, and writes a list of detected objects in the
# file output.
# Here we set the fwhm and sigma equal to those listed in the header.
# We set the datamin to -3*sigma = -1*threshold defined above
# The findpars.threshold is set to 4.5*sigma by default. You may have
# to alter this value for images where too few/many stars are found.
#Configure 'datapars', 'findpars', and 'daofind'
iraf.datapars.fwhmpsf=fwhm
iraf.datapars.sigma=sigma
iraf.datapars.datamax='indef'
iraf.datapars.datamin=(-1)*threshold
iraf.datapars.ccdread='RDNOISE'
iraf.datapars.gain="GAIN"
iraf.datapars.exposure="EXPTIME"
iraf.datapars.airmass="AIRMASS"
iraf.datapars.filter="FILTER"
iraf.datapars.obstime="TIME-OBS"
iraf.findpars.threshold=4.5
iraf.findpars.roundhi=3
iraf.findpars.roundlo=-3
iraf.daofind.verify='no'
iraf.daofind.verbose='no'
#create the variable for the coordinate file
allStars = imageName+'.coo'
#Remove a previous coordinate file if one exists
silentDelete(imageName+'.coo')
#Find the stars in the aligned R image
iraf.daofind("withsky",allStars)
print("The file containing the data of the stars in the aligned R-image is ",allStars)
print(" ")
if os.path.getsize(allStars) > 2239 :
print("B. Separate stars inside and outside galaxy")
#Delete existing files
for f in ("temp.file","maskfile","maskfile.sat"):
silentDelete(f)
#Extract the coordinates from the allStars file, redirecting stdout to do so
sys.stdout=open("temp.file","w")
iraf.pdump(allStars,"xcenter,ycenter",'yes')
sys.stdout = sys.__stdout__
#Read the file to get the xy coordinate pairs in a list
coords = open("temp.file")
coordList = list(coords)
countout=0
#Create strings of xy pairs to write to files
outGalString = ""
for pair in coordList:
#for each entry in the list, parse it into xy value integer pairs
values = pair.split(" ")
x = float(values[0])
y = float(values[1])
#Determine if the star lies within the galaxy-centered ellipse
inGalaxy = False
deltaX = x - xCenter
deltaY = y - yCenter
cdx = abs(deltaX)
cdy = abs(deltaY)
if (cdx < a and cdy < a):
xt=(deltaX*math.cos(posAngleRad))+(deltaY*math.sin(posAngleRad))
yt=(deltaY*math.cos(posAngleRad))-(deltaX*math.sin(posAngleRad))
if(xt <= a and xt >= na and yt <= b and yt >= nb):
inGalaxy = True
if (not inGalaxy):
outGalString += str(x) + " " + str(y) + "\n"
countout=countout+1
#Write the coordinate list to a file
mask = open("maskfile","w")
mask.write(outGalString)
mask.close()
print('')
print(" ",countout," stars found outside galaxy")
print('')
if countout!=0 :
print("C. Do photometry to find saturated stars")
for f in ("maskfile.mag","maskfile.satmag","maskfile.sat"):
silentDelete(f)
#Configure 'centerpars', 'fitskypars','photpars'
iraf.datapars.datamax=150000
iraf.datapars.datamin='indef'
iraf.centerpars.calgorithm="none"
iraf.fitskypars.salgorithm="mode"
iraf.fitskypars.annulus=annul
iraf.fitskypars.dannulus=10
iraf.photpars.apertures=fwhm
iraf.phot.verify='no'
iraf.phot.verbose='no'
#Call 'phot' to get the data of the stars
iraf.phot (imageName,"maskfile","maskfile.mag")
boexpr="PIER==305"
iraf.pselect("maskfile.mag","maskfile.satmag",boexpr)
sys.stdout=open("maskfile.sat","w")
iraf.pdump("maskfile.satmag","xcenter,ycenter","yes")
sys.stdout = sys.__stdout__
print("D. Make mask of stars outside galaxy")
#Delete the rmask if one exists
silentDelete("rmask.fits")
#Configure 'imedit'
iraf.imedit.cursor="maskfile"
iraf.imedit.display='no'
iraf.imedit.autodisplay='no'
iraf.imedit.aperture="circular"
iraf.imedit.value=-1000
iraf.imedit.default="e"
#Replace the pixel values near the star coordinates with value -1000 using imedit
iraf.imedit(imageName,"redit",radius=aper)
iraf.imcopy.verbose='no'
#Do the same for the saturated stars, but with larger apertures
satMaskSize = os.path.getsize("maskfile.sat")
if satMaskSize < 1 :
print(" No saturated stars found")
if (satMaskSize!=0):
iraf.imedit.cursor="maskfile.sat"
iraf.imedit.radius=2*aper
iraf.imedit("redit","rmask")
else:
iraf.imcopy("redit","rmask")
#Replace good pixels with value 0 in 'rmask'
iraf.imreplace.lower=-999
iraf.imreplace.upper='indef'
iraf.imreplace("rmask",0)
#Replace bad pixels with value 1 in 'rmask'
iraf.imreplace.lower='indef'
iraf.imreplace.upper=-1000
iraf.imreplace("rmask",1)
#Remove the mask file if one already exists
silentDelete(imageName+"mask.fits")
iraf.imcopy("rmask",imageName+"mask")
print(" ")
print("The mask image is ",imageName+"mask")
print("The masked image is ",imageName+"Masked")
print(" ")
else :
print("No stars found outside galaxy, no mask created.")
if os.path.getsize(allStars) < 2314 :
print("No stars found, no mask created.")
#Delete intermediate images
for f in ("rmask.fits","redit.fits","maskimage.fits","maskfile.mag","maskfile.satmag","temp.file","withsky.fits","mask.fits"):
silentDelete(f)
