def combine_cubes(listcubes, listmasks, regions=True):
"""
Combine cubes in mean mode o with median.
Apply masks as desired.
cubes -> a list of cubes to use in the combine
masks -> a list of goodpix masks from the pipeline
regions -> if True, code searches for ds9 region files inside path with same
name as pipeline mask (.reg), to mask additional area that one wants
to clip
"""
from astropy.io import fits
import numpy as np
import scipy
import os
import matplotlib.pyplot as plt
from mypython.fits import pyregmask as msk
if (os.path.isfile("COMBINED_CUBE_MED.fits")
& os.path.isfile("COMBINED_CUBE.fits")):
print("Coadded cubes already exists!")
return
#continue with implicit else if passed the checks
if (regions):
print("Updating the masks following ds9 regions")
#loads list
clistmask = np.loadtxt(listmasks, dtype=np.dtype('a'))
#redefine new mask
mask_new = "new_" + listmasks
llms = open(mask_new, "w")
#loop over and update with regions
#if scalar, make it 1 element list
if (clistmask.shape == ()):
clistmask = [clistmask]
for i, cmask in enumerate(clistmask):
#create region name
regname_line = (cmask.split(".fits")[0]) + ".reg"
#reconstruct cubex region name
rnpath = (cmask.split("MASK")[0])
rnexp = (cmask.split("_")[1])
regname_cubex = rnpath + "DATACUBE_FINAL_LINEWCS_" + rnexp + "_fix2_SliceEdgeMask.reg"
#search if file exist
if (os.path.isfile(regname_line)):
regname = regname_line
elif (os.path.isfile(regname_cubex)):
regname = regname_cubex
else:
regname = None
if (regname):
#update the mask
print("Updating mask using {}".format(regname))
#open fits
cfits = fits.open(cmask)
#init reg mask
Mask = msk.PyMask(cfits[1].header["NAXIS1"],
cfits[1].header["NAXIS2"], regname)
for ii in range(Mask.nreg):
Mask.fillmask(ii)
if (ii == 0):
totmask = Mask.mask
else:
totmask += Mask.mask
#update the mask
cfits[1].data = cfits[1].data * 1 * np.logical_not(totmask)
savename = cmask.split(".fits")[0] + '_wreg.fits'
cfits.writeto(savename, clobber=True)
llms.write(savename + '\n')
else:
#keep current mask
llms.write(cmask + '\n')
#done with new masks
llms.close()
else:
print('Using original masks...')
mask_new = listmasks
print("Combining cubes with mean and median")
#load the relevant lists
cblis = open(listcubes)
mklis = open(mask_new)
allcubes = []
allmasks = []
for cc in cblis:
allcubes.append(fits.open(cc.strip()))
for mm in mklis:
allmasks.append(fits.open(mm.strip()))
cblis.close()
mklis.close()
#generate list of cubes
nexp = len(allcubes)
print('Coadding {} exposures...'.format(nexp))
#make space for final grid
finalcube_mean = np.copy((allcubes[1])[1].data)
finalvar = np.copy((allcubes[1])[2].data)
finalcube_median = np.copy((allcubes[1])[1].data)
#grab info on pixels
nx = (allcubes[1])[1].header["NAXIS1"]
ny = (allcubes[1])[1].header["NAXIS2"]
nw = (allcubes[1])[1].header["NAXIS3"]
#giant for loop over wave,pix
print('Working on {} slices...'.format(nw))
piximage = np.zeros((nexp, ny, nx))
varimage = np.zeros((nexp, ny, nx))
mskimage = np.zeros((nexp, ny, nx))
masknans = np.zeros((ny, nx))
for ww in range(nw):
#print (' {} '.format(ww+1),end='')
#now loop over exposure
for ee in range(nexp):
piximage[ee, :] = (allcubes[ee])[1].data[ww, :]
varimage[ee, :] = (allcubes[ee])[2].data[ww, :]
#clean nan
masknans = masknans * 0
notnans = np.where(np.isfinite(piximage[ee, :]))
masknans[notnans] = 1
#1 good pixels at first, then 1 bad pixels
mskimage[ee, :] = np.logical_not(
((allmasks[ee])[1].data) * masknans)
#construct masked arrays
pixmasked = np.ma.array(piximage, mask=mskimage)
varmasked = np.ma.array(varimage, mask=mskimage)
#make coadds with masking
finalcube_median[ww, :] = np.ma.median(pixmasked, axis=0)
finalcube_mean[ww, :] = np.ma.mean(pixmasked, axis=0)
countmap = np.ma.count(varmasked, axis=0)
finalvar[ww, :] = np.ma.sum(varmasked, axis=0) / countmap / countmap
#write
hdu1 = fits.PrimaryHDU([])
hdu2 = fits.ImageHDU(finalcube_mean)
hdu3 = fits.ImageHDU(finalvar)
hdu2.header = (allcubes[0])[1].header
hdu3.header = (allcubes[0])[2].header
hdulist = fits.HDUList([hdu1, hdu2, hdu3])
hdulist.writeto("COMBINED_CUBE.fits", clobber=True)
#write
hdu1 = fits.PrimaryHDU([])
hdu2 = fits.ImageHDU(finalcube_median)
hdu3 = fits.ImageHDU(finalvar)
hdu2.header = (allcubes[0])[1].header
hdu3.header = (allcubes[0])[2].header
hdulist = fits.HDUList([hdu1, hdu2, hdu3])
hdulist.writeto("COMBINED_CUBE_MED.fits", clobber=True)
#make white images
print('Creating final white images')
white_mean = np.zeros((ny, nx))
white_med = np.zeros((ny, nx))
for xx in range(nx):
for yy in range(ny):
white_mean[yy, xx] = np.sum(finalcube_mean[:, yy, xx]) / nw
white_med[yy, xx] = np.sum(finalcube_median[:, yy, xx]) / nw
#save projected image
hdu1 = fits.PrimaryHDU([])
hdu2 = fits.ImageHDU(white_mean)
hdu2.header = (allcubes[0])[1].header
hdulist = fits.HDUList([hdu1, hdu2])
hdulist.writeto("COMBINED_IMAGE.fits", clobber=True)
#save projected image
hdu1 = fits.PrimaryHDU([])
hdu2 = fits.ImageHDU(white_med)
hdu2.header = (allcubes[0])[1].header
hdulist = fits.HDUList([hdu1, hdu2])
hdulist.writeto("COMBINED_IMAGE_MED.fits", clobber=True)
def internalskysub(listob, skymask, deepwhite=None):
"""
Perform sky-subtraction using pixels within the cube
listob -> OBs to loop on
skymask -> if defined to a ds9 region file (iamge coordinate),
compute sky in these regions (excluding sources)
Otherwise mask sources and use all the pixels in the field.
"""
import os
import glob
from astropy.io import fits
import numpy as np
import zap
import matplotlib.pyplot as plt
import sep
#grab top dir
topdir = os.getcwd()
#now loop over each folder and make the final illcorrected cubes
for ob in listob:
#change dir
os.chdir(ob + '/Proc/Line/')
print('Processing {} for sky subtraction correction'.format(ob))
#Search how many exposures are there
scils = glob.glob("../Basic/OBJECT_RED_0*.fits*")
nsci = len(scils)
#loop on exposures and reduce frame with zeroth order sky subtraction + ZAP
for exp in range(nsci):
#do pass on IFUs
print('Interal sky subtraction of exposure {}'.format(exp + 1))
#define names
oldcube = "DATACUBE_FINAL_LINEWCS_EXP{0:d}_ILLCORR_stack.fits".format(
exp + 1)
oldimage = "IMAGE_FOV_LINEWCS_EXP{0:d}_ILLCORR_stack.fits".format(
exp + 1)
newcube = "DATACUBE_FINAL_LINEWCS_EXP{0:d}_lineskysub.fits".format(
exp + 1)
newimage = "IMAGE_FOV_LINEWCS_EXP{0:d}_lineskysub.fits".format(
exp + 1)
ifumask_iname = "IMAGE_IFUMASK_LINEWCS_EXP{0:d}.fits".format(exp +
1)
source_mask = "IMAGE_SOURCEMASK_LINEWCS_EXP{0:d}.fits".format(exp +
1)
zapcube = "DATACUBE_FINAL_LINEWCS_EXP{0:d}_zapsky.fits".format(
exp + 1)
zapimage = "IMAGE_FOV_LINEWCS_EXP{0:d}_zapsky.fits".format(exp + 1)
zapsvdout = "ZAPSVDOUT_EXP{0:d}.fits".format(exp + 1)
if not os.path.isfile(zapcube):
#open the cube
cube = fits.open(oldcube)
#open mask ifu
ifumask = fits.open(ifumask_iname)
#if white image provided load it
if (deepwhite):
print("Use source mask image {}".format(deepwhite))
whsrc = fits.open(topdir + '/' + deepwhite)
whitesource = whsrc[0].data.byteswap().newbyteorder()
else:
#create from cube
print("Create source mask image from cube")
whitesource = np.nanmedian(cube[1].data, axis=0)
#now create a source mask
print('Create a source mask')
header = cube[1].header
bkg = sep.Background(whitesource)
bkg_subtraced_data = whitesource - bkg.back()
thresh = 3. * bkg.globalrms
minarea = 20.
clean = True
segmap = np.zeros((header["NAXIS2"], header["NAXIS1"]))
#extract objects
objects, segmap = sep.extract(bkg_subtraced_data,
thresh,
segmentation_map=True,
minarea=minarea,
clean=clean)
#plt.imshow(segmap,origin='low')
#plt.show()
#plt.imshow(whitesource,origin='low')
#plt.show()
#define geometry
nwave = cube[1].header["NAXIS3"]
nx = cube[1].header["NAXIS1"]
ny = cube[1].header["NAXIS2"]
#make sure pixels are sky sub once and only once
countsub = np.copy(ifumask[1].data) * 0.
#if mask is set do a corse median sky subtraction
if (skymask):
print('Constructing sky mask')
#for zap, sky region should be 0, and sources >1
skybox = np.zeros((ny, nx)) + 1
#construct the sky region mask
from mypython.fits import pyregmask as pmk
mysky = pmk.PyMask(nx,
ny,
"../../../" + skymask,
header=cube[1].header)
for ii in range(mysky.nreg):
mysky.fillmask(ii)
usepix = np.where(mysky.mask > 0)
skybox[usepix] = 0
#plt.imshow(skybox,origin='low')
#plt.show()
#plt.imshow(segmap,origin='low')
#plt.show()
#plt.imshow(ifumask[1].data,origin='low')
#plt.show()
#exit()
#now do median sky subtraction
#loop over wavelength
for ww in range(nwave):
#extract sky slice
skyimg = cube[1].data[ww, :, :]
#grab pixels with no source and in mask region
#avoid edges not flagged by IFU mask
pixels = np.where((skybox < 1) & (segmap < 1)
& (ifumask[1].data > 0))
#compute sky in good regions
medsky = np.nanmedian(skyimg[pixels])
#subtract from all pixels
cube[1].data[ww, :, :] = skyimg - medsky
else:
#otherwise do coarse sky IFU by IFU
#loop over ifu
for iff in range(24):
thisifu = (iff + 1) * 100.
nextifu = (iff + 2) * 100. + 1
#grab pixels in ifu without sources
pixels=np.where((ifumask[1].data >= thisifu) & \
(ifumask[1].data < nextifu)\
& (segmap < 1) )
pixels_ifu=np.where((ifumask[1].data >= thisifu) \
& (ifumask[1].data < nextifu)\
& (countsub < 1))
#update used pixels
countsub[pixels_ifu] = 1
#loop over wavelength
for ww in range(nwave):
skyimg = cube[1].data[ww, :, :]
#compute sky in good regions
medsky = np.nanmedian(skyimg[pixels])
#subtract from all IFU pixels
skyimg[pixels_ifu] = skyimg[pixels_ifu] - medsky
cube[1].data[ww, :, :] = skyimg
#write final cube
cube.writeto(newcube, clobber=True)
#create white image
print('Creating final white image')
white_new = np.zeros((ny, nx))
for xx in range(nx):
for yy in range(ny):
white_new[yy, xx] = np.nansum(cube[1].data[:, yy,
xx]) / nwave
#save projected image
hdu1 = fits.PrimaryHDU([])
hdu2 = fits.ImageHDU(white_new)
hdu2.header = cube[1].header
hdulist = fits.HDUList([hdu1, hdu2])
hdulist.writeto(newimage, clobber=True)
#save segmap
#make it redundant to be sure ZAP read right extension
hdu1 = fits.PrimaryHDU(segmap)
#hdu1.header=header
hdu2 = fits.ImageHDU(segmap)
#hdu2.header=header
hdulist = fits.HDUList([hdu1, hdu2])
hdulist.writeto(source_mask, clobber=True)
print('Running ZAP on exposure {}'.format(exp + 1))
#deal with masks
if (skymask):
#combine sky mask with source mask
#make it redundant to be sure ZAP read right extension
tmpzapmask = segmap + skybox
hdu1 = fits.PrimaryHDU(tmpzapmask)
#hdu1.header=header
hdu2 = fits.ImageHDU(tmpzapmask)
#hdu2.header=header
hdulist = fits.HDUList([hdu1, hdu2])
hdulist.writeto("ZAP_" + source_mask, clobber=True)
zapmask = "ZAP_" + source_mask
else:
zapmask = source_mask
#clean old if exists
try:
os.remove(zapsvdout)
except:
pass
#run new - handle change in keywords from v1 to v2
try:
zap.process(newcube,
outcubefits=zapcube,
clean=True,
svdoutputfits=zapsvdout,
mask=zapmask)
except:
zap.process(newcube,
outcubefits=zapcube,
clean=True,
mask=zapmask)
#create white image from zap cube
cube = fits.open(zapcube)
print('Creating final white image from ZAP')
white_new = np.zeros((ny, nx))
for xx in range(nx):
for yy in range(ny):
white_new[yy, xx] = np.nansum(cube[1].data[:, yy,
xx]) / nwave
#save projected image
hdu1 = fits.PrimaryHDU([])
hdu2 = fits.ImageHDU(white_new)
hdu2.header = cube[1].header
hdulist = fits.HDUList([hdu1, hdu2])
hdulist.writeto(zapimage, clobber=True)
else:
print("ZAP cube exist alread for exposure {}... skip!".format(
exp + 1))
#back to top for next OB
os.chdir(topdir)
def findsources(image,
cube,
varima=None,
check=False,
output='./',
spectra=False,
helio=0,
nsig=2.,
minarea=10.,
deblend_cont=0.0001,
regmask=None,
invregmask=False,
fitsmask=None,
clean=True,
outspec='Spectra',
marz=False,
rphot=False,
detphot=False,
sname='MUSE'):
"""
Take a detection image (collapse of a cube), or median
of an RGB, or whatever you want (but aligned to the cube)
and run sourceextractor
Use SEP utilities http://sep.readthedocs.org/en/stable/
image -> fits file of image to process
cube -> the cube used to extract spectra
varima -> the noise image corresponding to the science image (std), optional
check -> if true write a bunch of check mages
output -> where to dump the output
spectra -> if True, extract spectra in VACUUM wave!!
helio -> pass additional heliocentric correction
nsig -> number of skyrms used for source id
minarea -> minimum area for extraction
regmask -> ds9 region file (image) of regions to be masked before extraction [e.g. edges]
invregmask -> if True invert the mask (region defines good area)
fitsmask -> Fits file with good mask, overrides regmask
clean -> clean souces
outspec -> where to store output spectra
marz -> write spectra in also marz format (spectra needs to be true).
If set to numerical value, this is used as an r-band magnitude limit.
detphot -> perform aperture phtometry on the detection image and add magnitues to the catalogue
rphot -> perform r-band aperture photometry and add r-band magnitudes to the catalogue
sname -> prefix for the source names. Default = MUSE
"""
import sep
from astropy.io import fits
from astropy import wcs
from astropy import coordinates
from astropy import units as u
from astropy import table
import numpy as np
import os
from mypython.ifu import muse_utils as utl
from mypython.fits import pyregmask as msk
from shutil import copyfile
import glob
#open image
img = fits.open(image)
header = img[0].header
imgwcs = wcs.WCS(header)
try:
#this is ok for narrow band images
data = img[1].data
except:
#white cubex images
data = img[0].data
data = data.byteswap(True).newbyteorder()
#grab effective dimension
nex, ney = data.shape
#close fits
img.close()
if (varima):
var = fits.open(varima)
try:
datavar = var[1].data
except:
datavar = var[0].data
datavar = datavar.byteswap(True).newbyteorder()
#grab effective dimension
stdx, stdy = datavar.shape
#close fits
var.close()
if (stdx != nex) or (stdy != ney):
print(
"The noise image does not have the same dimensions as the science image"
)
return -1
#create bad pixel mask
if (fitsmask):
print("Using FITS image for badmask")
hdumsk = fits.open(fitsmask)
try:
badmask = hdumsk[1].data
except:
badmask = hdumsk[0].data
badmask = badmask.byteswap(True).newbyteorder()
elif (regmask):
print("Using region file for badmask")
Mask = msk.PyMask(ney, nex, regmask, header=img[0].header)
for ii in range(Mask.nreg):
Mask.fillmask(ii)
if (ii == 0):
badmask = Mask.mask
else:
badmask += Mask.mask
badmask = 1. * badmask
else:
badmask = np.zeros((nex, ney))
if (regmask) and (invregmask) and not (fitsmask):
badmask = 1 - badmask
if (check):
print('Dumping badmask')
hdumain = fits.PrimaryHDU(badmask, header=header)
hdulist = fits.HDUList([hdumain])
hdulist.writeto(output + "/badmask.fits", overwrite=True)
#check background level, but do not subtract it
print('Checking background levels')
bkg = sep.Background(data, mask=badmask)
print('Residual background level ', bkg.globalback)
print('Residual background rms ', bkg.globalrms)
if (check):
print('Dumping sky...')
#dump sky properties
back = bkg.back()
rms = bkg.rms()
hdumain = fits.PrimaryHDU(back, header=header)
hdubk = fits.ImageHDU(back)
hdurms = fits.ImageHDU(rms)
hdulist = fits.HDUList([hdumain, hdubk, hdurms])
hdulist.writeto(output + "/skyprop.fits", overwrite=True)
if (varima):
#Use nsigma threshold and a pixel by pixel effective treshold based on variance map
thresh = nsig
objects, segmap = sep.extract(data,
thresh,
var=datavar,
segmentation_map=True,
minarea=minarea,
clean=clean,
mask=badmask,
deblend_cont=deblend_cont,
deblend_nthresh=32)
else:
#extracting sources at nsigma, use constant threshold
thresh = nsig * bkg.globalrms
objects, segmap = sep.extract(data,
thresh,
segmentation_map=True,
minarea=minarea,
clean=clean,
mask=badmask,
deblend_cont=deblend_cont,
deblend_nthresh=32)
print("Extracted {} objects... ".format(len(objects)))
ids = np.arange(len(objects)) + 1
if (spectra):
if not os.path.exists(outspec):
os.makedirs(outspec)
if ((check) | (spectra)):
#create a detection mask a'la cubex
srcmask = np.zeros((data.shape[0], data.shape[1]))
print('Generating spectra...')
#loop over detections
for nbj in ids:
obj = objects[nbj - 1]
#init mask
tmpmask = np.zeros((data.shape[0], data.shape[1]), dtype=np.bool)
#fill this mask
sep.mask_ellipse(tmpmask,
obj['x'],
obj['y'],
obj['a'],
obj['b'],
obj['theta'],
r=2)
#add in global mask
srcmask = srcmask + tmpmask * nbj
#verify conflicts, resolve using segmentation map
if np.nanmax(srcmask) > nbj:
blended = (srcmask > nbj)
srcmask[blended] = segmap[blended]
#Now loop again and extract spectra if required
if (spectra):
#Verify that the source mask has the same number of objects as the object list
if not len(np.unique(srcmask[srcmask > 0])) == len(objects):
print(
"Mismatch between number of objects and number of spectra to extract."
)
for nbj in ids:
savename = "{}/id{}.fits".format(outspec, nbj)
tmpmask3d = np.zeros((1, data.shape[0], data.shape[1]))
tmpmask3d[0, :, :] = srcmask[:, :]
tmpmask3d[tmpmask3d != nbj] = 0
tmpmask3d[tmpmask3d > 0] = 1
tmpmask3d = np.array(tmpmask3d, dtype=np.bool)
utl.cube2spec(cube,
None,
None,
None,
write=savename,
shape='mask',
helio=helio,
mask=tmpmask3d,
tovac=True)
if (check):
print('Dumping source mask...')
hdumain = fits.PrimaryHDU(srcmask, header=header)
hdubk = fits.ImageHDU(srcmask)
hdulist = fits.HDUList([hdumain, hdubk])
hdulist.writeto(output + "/source.fits", overwrite=True)
print('Dumping segmentation map')
hdumain = fits.PrimaryHDU(segmap, header=header)
hdubk = fits.ImageHDU(segmap)
hdulist = fits.HDUList([hdumain, hdubk])
hdulist.writeto(output + "/segmap.fits", overwrite=True)
#Generate source names using coordinates and name prefix
ra, dec = imgwcs.wcs_pix2world(objects['x'], objects['y'], 0)
coord = coordinates.FK5(ra * u.degree, dec * u.degree)
rastr = coord.ra.to_string(u.hour, precision=2, sep='', pad=True)
decstr = coord.dec.to_string(u.degree,
precision=1,
sep='',
alwayssign=True,
pad=True)
name = [
sname + 'J{0}{1}'.format(rastr[k], decstr[k])
for k in range(len(rastr))
]
#Generate a column to be used to flag the sources to be used in the analysis
#True for all sources at this point
use_source = np.ones_like(name, dtype=bool)
#write source catalogue
print('Writing catalogue..')
tab = table.Table(objects)
tab.add_column(table.Column(dec), 0, name='DEC')
tab.add_column(table.Column(ra), 0, name='RA')
tab.add_column(table.Column(name), 0, name='name')
tab.add_column(table.Column(ids), 0, name='ID')
tab.add_column(table.Column(use_source), name='use_source')
tab.write(output + '/catalogue.fits', overwrite=True)
if (detphot):
#Run source photometry on the extraction image
whiteimg, whitevar, whitewcsimg = utl.cube2img(cube,
write=output +
'/Image_white.fits')
phot_det = sourcephot(output + '/catalogue.fits',
output + '/Image_white.fits',
output + '/segmap.fits',
image,
zpab=28.35665)
phot_det.add_column(table.Column(name), 1, name='name')
tbhdu = fits.open(output + '/catalogue.fits')
tbhdu.append(fits.BinTableHDU(phot_det))
tbhdu[-1].header['PHOTBAND'] = 'Detection'
tbhdu.writeto(output + '/catalogue.fits', overwrite=True)
#rband photometry
if (rphot):
rimg, rvar, rwcsimg = utl.cube2img(cube,
filt=129,
write=output + '/Image_R.fits')
phot_r = sourcephot(output + '/catalogue.fits',
output + '/Image_R.fits', output + '/segmap.fits',
image)
phot_r.add_column(table.Column(name), 1, name='name')
tbhdu = fits.open(output + '/catalogue.fits')
tbhdu.append(fits.BinTableHDU(phot_r))
tbhdu[-1].header['PHOTBAND'] = 'SDSS_r'
tbhdu.writeto(output + '/catalogue.fits', overwrite=True)
if ((marz) & (spectra)):
#if marz is True but no magnitude limit set, create marz file for whole catalogue
if marz > 10 and (rphot):
#Requires testing
hdu = fits.open(output + '/catalogue.fits')
hdu[1].data['use_source'][hdu[2].data['MAGAP'] > marz] = False
hdu.writeto(output + '/catalogue.fits', overwrite=True)
marz_file(output + '/catalogue.fits', outspec, output, r_lim=marz)
else:
marz_file(output + '/catalogue.fits', outspec, output)
print('All done')
return objects
def findsources(image,cube,check=False,output='.',spectra=False,helio=0,nsig=2.,
minarea=10.,regmask=None,clean=True,outspec='Spectra',marz=False,
rphot=False, sname='MUSE'):
"""
Take a detection image (collapse of a cube), or median
of an RGB, or whatever you want (but aligned to the cube)
and run sourceextractor
Use SEP utilities http://sep.readthedocs.org/en/stable/
image -> fits file of image to process
check -> if true write a bunch of check mages
output -> where to dump the output
cube -> the cube used to extract spectra
spectra -> if True, extract spectra in VACUUM wave!!
helio -> pass additional heliocentric correction
nsig -> number of skyrms used for source id
minarea -> minimum area for extraction
regmask -> ds9 region file (image) of regions to be masked before extraction [e.g. edges]
clean -> clean souces
outspec -> where to store output spectra
marz -> write spectra in also marz format (spectra needs to be true).
If set to numerical value, this is used as an r-band magnitude limit.
rphot -> perform r-band aperture photometry and add r-band magnitudes to the catalogue
sname -> prefix for the source names. Default = MUSE
"""
import sep
from astropy.io import fits
from astropy import wcs
from astropy import coordinates
from astropy import units as u
from astropy import table
import numpy as np
import os
try:
from mypython.ifu import muse_utils as utl
from mypython.fits import pyregmask as msk
except ImportError:
from mypython import ifu
from ifu import muse_utils as utl
from mypython import fits
from fits import pyregmask as msk
from astropy.io import fits
from shutil import copyfile
import glob
#open image
img=fits.open(image)
try:
header=img[1].header
except:
header= img[0].header
imgwcs = wcs.WCS(header)
try:
#this is ok for narrow band images
data=img[1].data
except:
#white cubex images
data=img[0].data
data=data.byteswap(True).newbyteorder()
#grab effective dimension
nex,ney=data.shape
#close fits
img.close()
#create bad pixel mask
if(regmask):
Mask=msk.PyMask(ney,nex,regmask,header=img[0].header)
for ii in range(Mask.nreg):
Mask.fillmask(ii)
if(ii == 0):
badmask=Mask.mask
else:
badmask+=Mask.mask
badmask=1.*badmask
else:
badmask=np.zeros((nex,ney))
if(check):
print('Dumping badmask')
hdumain = fits.PrimaryHDU(badmask,header=header)
hdulist = fits.HDUList([hdumain])
hdulist.writeto(output+"/badmask.fits",overwrite=True)
#check background level, but do not subtract it
print('Checking background levels')
bkg = sep.Background(data,mask=badmask)
print('Residual background level ', bkg.globalback)
print('Residual background rms ', bkg.globalrms)
if(check):
print('Dumping sky...')
#dump sky properties
back = bkg.back()
rms = bkg.rms()
hdumain = fits.PrimaryHDU(back,header=header)
hdubk = fits.ImageHDU(back)
hdurms = fits.ImageHDU(rms)
hdulist = fits.HDUList([hdumain,hdubk,hdurms])
hdulist.writeto(output+"/skyprop.fits",overwrite=True)
#extracting sources at nsigma
thresh = nsig * bkg.globalrms
# segmap = np.zeros((header["NAXIS1"],header["NAXIS2"]))
objects, segmap=sep.extract(data,thresh,segmentation_map=True,
minarea=minarea,clean=clean,mask=badmask,deblend_cont=0.0001)
print("Extracted {} objects... ".format(len(objects)))
if(spectra):
if not os.path.exists(outspec):
os.makedirs(outspec)
if((check) | (spectra)):
#create a detection mask alla cubex
srcmask=np.zeros((1,data.shape[0],data.shape[1]))
nbj=1
print('Generating spectra...')
#loop over detections
for obj in objects:
#init mask
tmpmask=np.zeros((data.shape[0],data.shape[1]),dtype=np.bool)
tmpmask3d=np.zeros((1,data.shape[0],data.shape[1]),dtype=np.bool)
#fill this mask
sep.mask_ellipse(tmpmask,obj['x'],obj['y'],obj['a'],obj['b'],obj['theta'],r=2)
tmpmask3d[0,:,:]=tmpmask[:,:]
srcmask=srcmask+tmpmask3d*nbj
if(spectra):
savename="{}/id{}.fits".format(outspec,nbj)
if not os.path.exists(savename):
utl.cube2spec(cube,obj['x'],obj['y'],None,write=savename,
shape='mask',helio=helio,mask=tmpmask3d,tovac=True)
else:
print("{} already exists. Skipping it...".format(savename))
#go to next
nbj=nbj+1
if(check):
print('Dumping source mask...')
hdumain = fits.PrimaryHDU(srcmask,header=header)
hdubk = fits.ImageHDU(srcmask)
hdulist = fits.HDUList([hdumain,hdubk])
hdulist.writeto(output+"/source.fits",overwrite=True)
print('Dumping segmentation map')
hdumain = fits.PrimaryHDU(segmap,header=header)
hdubk = fits.ImageHDU(segmap)
hdulist = fits.HDUList([hdumain,hdubk])
hdulist.writeto(output+"/segmap.fits",overwrite=True)
#Generate source names using coordinates and name prefix
ra, dec = imgwcs.wcs_pix2world(objects['x'], objects['y'],0)
coord = coordinates.FK5(ra*u.degree, dec*u.degree)
rastr = coord.ra.to_string(u.hour, precision=2, sep='')
decstr = coord.dec.to_string(u.degree, precision=1, sep='', alwayssign=True)
name = [sname+'J{0}{1}'.format(rastr[k], decstr[k]) for k in range(len(rastr))]
ids = np.arange(len(name))
#write source catalogue
print('Writing catalogue..')
tab = table.Table(objects)
tab.add_column(table.Column(name),0,name='name')
tab.add_column(table.Column(ids),0,name='ID')
tab.write(output+'/catalogue.fits',overwrite=True)
#cols = fits.ColDefs(objects)
#cols.add_col(fits.Column(name, format='A'))
#tbhdu = fits.BinTableHDU.from_columns(cols)
#tbhdu.writeto(output+'/catalogue.fits',clobber=True)
#rband photometry
if (rphot):
if not os.path.exists(output+'/Image_R.fits'):
rimg, rvar, rwcsimg = utl.cube2img(cube, filt=129, write=output+'/Image_R.fits')
phot_r = sourcephot(output+'/catalogue.fits', output+'/Image_R.fits', output+'/segmap.fits', image)
phot_r.add_column(table.Column(name),1,name='name')
tbhdu = fits.open(output+'/catalogue.fits')[1]
tbhdu2 = fits.BinTableHDU(phot_r)
hdulist = fits.HDUList([fits.PrimaryHDU(), tbhdu, tbhdu2])
hdulist.writeto(output+'/catalogue.fits',overwrite=True)
if((marz) & (spectra)):
#if marz is True but no magnitude limit set, create marz file for whole catalogue
if marz==True:
marz_file(image, output+'/catalogue.fits', outspec, output)
else:
#create folder and catalogue with just sources brighter than mag limit
if os.path.exists(output + '/spectra_r' + str(marz)):
files = glob.glob(output + '/spectra_r' +
str(marz) +'/*')
for f in files:
os.remove(f)
else:
os.mkdir(output + '/spectra_r' + str(marz))
mag = phot_r['MAGSEG']
#add in x y pixels from original catalogue
x, y = tbhdu.data['x'], tbhdu.data['y']
phot_r['x'], phot_r['y'] = x, y
#add in ra,dec
img = fits.open(image)
mywcs = wcs.WCS(img[0].header)
ra, dec = mywcs.all_pix2world(x,y,0)
phot_r['RA'] = ra
phot_r['dec'] = dec
for i in range(len(mag)):
if mag[i] < marz:
copyfile((output + '/spectra/id' + str(i+1)
+ '.fits'), (output + '/spectra_r' +
str(marz) + '/id' + str(i+1) + '.fits'))
#Write photometry catalog with objects below magnitude limit excluded
phot_r.remove_rows(phot_r['MAGSEG'] > marz)
catalogue_lim_name = (output + '/catalogue_r' +
str(marz) +'.fits')
if os.path.exists(catalogue_lim_name):
os.remove(catalogue_lim_name)
phot_r.write(catalogue_lim_name)
outspec = output + '/spectra_r' + str(marz)
marz_file(image, output+'/catalogue_r' + str(marz) +'.fits', outspec, output, r_lim=marz)
print('All done')
return objects
def combine_cubes(cubes, masks, regions=True, final=False):
"""
Combine a bunch of cubes using masks with CubeCombine
cubes -> a list of cubes to use in the combine
masks -> a list of goodpix masks from the pipeline
regions -> if True, code searches for ds9 region files inside path with same name as
pipeline mask (.reg), to mask additional area that one wants to clip
final -> is True, append final tag to name and prepare median cubes
"""
import subprocess
import os
import numpy as np
from astropy.io import fits
from mypython.fits import pyregmask as msk
#define some names for the cubes
if (final):
cname = "COMBINED_CUBE_FINAL.fits"
iname = "COMBINED_IMAGE_FINAL.fits"
cmed = "COMBINED_CUBE_MED_FINAL.fits"
imed = "COMBINED_IMAGE_MED_FINAL.fits"
else:
cname = "COMBINED_CUBE.fits"
iname = "COMBINED_IMAGE.fits"
cmed = "COMBINED_CUBE_MED.fits"
imed = "COMBINED_IMAGE_MED.fits"
if (os.path.isfile(cname)):
print('Cube {} already exists... skip!'.format(cname))
else:
print('Creating combined cube {}'.format(cname))
if (regions):
print "Updating the masks"
#loads list
listmask = np.loadtxt(masks, dtype=np.dtype('a'))
#redefine new mask
mask_new = "new_" + masks
llms = open(mask_new, "w")
#loop over and update with regions
for i, cmask in enumerate(listmask):
#create region name
regname = (cmask.split(".fits")[0]) + ".reg"
#search if file exist
if (os.path.isfile(regname)):
#update the mask
print("Updating mask for {}".format(regname))
#open fits
cfits = fits.open(cmask)
#init reg mask
Mask = msk.PyMask(cfits[0].header["NAXIS1"],
cfits[0].header["NAXIS2"], regname)
for ii in range(Mask.nreg):
Mask.fillmask(ii)
if (ii == 0):
totmask = Mask.mask
else:
totmask += Mask.mask
#update the mask
cfits[0].data = cfits[0].data * 1 * np.logical_not(totmask)
savename = cmask.split(".fits")[0] + '_wreg.fits'
cfits.writeto(savename, clobber=True)
llms.write(savename + '\n')
else:
#keep current mask
llms.write(cmask + '\n')
#done with new masks
llms.close()
else:
print 'Using original masks'
mask_new = masks
#now run combine
print 'Combine the cube...'
#make mean cube - write this as script that can be ran indepedently
if (final):
scriptname = 'runcombine_final.sh'
else:
scriptname = 'runcombine.sh'
scr = open(scriptname, 'w')
scr.write("export OMP_NUM_THREADS=1\n")
scr.write("CubeCombine -list " + cubes + " -out " + cname +
" -masklist " + mask_new + "\n")
scr.write("Cube2Im -cube " + cname + " -out " + iname + "\n")
scr.write("CubeCombine -list " + cubes + " -out " + cmed +
" -masklist " + mask_new + " -comb median\n")
scr.write("Cube2Im -cube " + cmed + " -out " + imed)
scr.close()
subprocess.call(["sh", scriptname])
def fixandsky_secondpass(cube, pixtab, noclobber, highsn=None, skymask=None):
"""
Similar to first pass, but operates on cubes that have been realigned and uses masks
as appropriate
If highsn cube is provided, use it to mask the sources and for skysubtraction
"""
import os
import subprocess
import numpy as np
from astropy.io import fits
from mypython.fits import pyregmask as pmk
if (highsn):
#prepare final names
fixed = cube.split('.fits')[0] + "_fixhsn.fits"
skysub = cube.split('.fits')[0] + "_skysubhsn.fits"
white = cube.split('.fits')[0] + "_whitehsn.fits"
sharpmsk = cube.split('.fits')[0] + "_sharpmasksn.fits"
else:
#prepare intermediate names
fixed = cube.split('.fits')[0] + "_fix2.fits"
skysub = cube.split('.fits')[0] + "_skysub2.fits"
white = cube.split('.fits')[0] + "_white2.fits"
sharpmsk = cube.split('.fits')[0] + "_sharpmask2.fits"
#assign names for source mask
mask_source = cube.split('.fits')[0] + "_white.Objects_Id.fits"
white_source = cube.split('.fits')[0] + "_white.fits"
#now fix the cube using masks
if ((os.path.isfile(fixed)) & (noclobber)):
print "Cube {0} already fixed".format(cube)
else:
print 'Create source mask ', white_source
#if high cube provide, overwrite white image
if (highsn):
print 'Using high SN cube...'
subprocess.call(["Cube2Im", "-cube", highsn, "-out", white_source])
subprocess.call([
"CubEx", white_source, '-MultiExt', '.false.', '-SN_Threshold',
'3', '-RescaleVar', '.true.'
])
else:
print 'Using white image from previous loop'
#create source mask
subprocess.call([
"CubEx", white_source, '-MultiExt', '.false.', '-SN_Threshold',
'5', '-RescaleVar', '.true.'
])
print 'Cubefix ', cube
subprocess.call([
"CubeFix", "-cube", cube, "-pixtable", pixtab, "-out", fixed,
"-sourcemask", mask_source
])
#At this step, check out cubeAdd2Mask if want to fix edges or weird ifus/slices
#if told to mask sky do it.. otherwise leave image empty
cb = fits.open(cube)
nx = cb[1].header['NAXIS1']
ny = cb[1].header['NAXIS2']
sharpmask = np.zeros((ny, nx))
if (skymask):
#construct the sky region mask
mysky = pmk.PyMask(nx, ny, "../../" + skymask, header=cb[1].header)
for ii in range(mysky.nreg):
mysky.fillmask(ii)
sharpmask = sharpmask + mysky.mask
cb.close()
#inject src mask in sharpmask
srcmk = fits.open(mask_source)
sharpmask = sharpmask + srcmk[0].data
#write mask
hdu = fits.PrimaryHDU(sharpmask)
hdulist = fits.HDUList([hdu])
hdulist.writeto(sharpmsk, clobber=True)
#now run cube skysub
if ((os.path.isfile(skysub)) & (noclobber)):
print "Cube {0} already skysub".format(fixed)
else:
print 'Sky sub ', fixed
if (highsn):
#now few more options to control sky sub
subprocess.call([
"CubeSharp", "-cube", fixed, "-out", skysub, "-sourcemask",
sharpmsk, "-hsncube", highsn, "-lcheck", ".false."
])
else:
subprocess.call([
"CubeSharp", "-cube", fixed, "-out", skysub, "-sourcemask",
sharpmsk
])
#create a white image
if ((os.path.isfile(white)) & (noclobber)):
print "White image for cube {0} already exists".format(skysub)
else:
print 'Create white image for ', skysub
subprocess.call(["Cube2Im", "-cube", skysub, "-out", white])
def findsources(image,
cube,
check=False,
output='./',
spectra=False,
helio=0,
nsig=2.,
minarea=10.,
regmask=None,
clean=True,
outspec='Spectra'):
"""
Take a detection image (collapse of a cube), or median
of an RGB, or whatever you want (but aligned to the cube)
and run sourceextractor
Use SEP utilities http://sep.readthedocs.org/en/stable/
image -> fits file of image to process
check -> if true write a bunch of check mages
output -> where to dump the output
cube -> the cube used to extract spectra
spectra -> if True, extract spectra in VACUUM wave!!
helio -> pass additional heliocentric correction
nsig -> number of skyrms used for source id
minarea -> minimum area for extraction
regmask -> ds9 region file (image) of regions to be masked before extraction [e.g. edges]
clean -> clean souces
outspec -> where to store output spectra
"""
import sep
from astropy.io import fits
import numpy as np
import os
from mypython.ifu import muse_utils as utl
from mypython.fits import pyregmask as msk
#open image
img = fits.open(image)
header = img[0].header
try:
#this is ok for narrow band images
data = img[1].data
except:
#white cubex images
data = img[0].data
data = data.byteswap(True).newbyteorder()
#grab effective dimension
nex, ney = data.shape
#close fits
img.close()
#create bad pixel mask
if (regmask):
Mask = msk.PyMask(ney, nex, regmask)
for ii in range(Mask.nreg):
Mask.fillmask(ii)
if (ii == 0):
badmask = Mask.mask
else:
badmask += Mask.mask
badmask = 1. * badmask
else:
badmask = np.zeros((nex, ney))
if (check):
print('Dumping badmask')
hdumain = fits.PrimaryHDU(badmask, header=header)
hdulist = fits.HDUList([hdumain])
hdulist.writeto(output + "/badmask.fits", clobber=True)
#check background level, but do not subtract it
print 'Checking background levels'
bkg = sep.Background(data, mask=badmask)
print 'Residual background level ', bkg.globalback
print 'Residual background rms ', bkg.globalrms
if (check):
print 'Dumping sky...'
#dump sky properties
back = bkg.back()
rms = bkg.rms()
hdumain = fits.PrimaryHDU(back, header=header)
hdubk = fits.ImageHDU(back)
hdurms = fits.ImageHDU(rms)
hdulist = fits.HDUList([hdumain, hdubk, hdurms])
hdulist.writeto(output + "/skyprop.fits", clobber=True)
#extracting sources at nsigma
thresh = nsig * bkg.globalrms
segmap = np.zeros((header["NAXIS1"], header["NAXIS2"]))
objects, segmap = sep.extract(data,
thresh,
segmentation_map=True,
minarea=minarea,
clean=clean,
mask=badmask)
print "Extracted {} objects... ".format(len(objects))
if (spectra):
if not os.path.exists(outspec):
os.makedirs(outspec)
if ((check) | (spectra)):
#create a detection mask alla cubex
srcmask = np.zeros((1, data.shape[0], data.shape[1]))
nbj = 1
print('Generating spectra...')
#loop over detections
for obj in objects:
#init mask
tmpmask = np.zeros((data.shape[0], data.shape[1]), dtype=np.bool)
tmpmask3d = np.zeros((1, data.shape[0], data.shape[1]),
dtype=np.bool)
#fill this mask
sep.mask_ellipse(tmpmask,
obj['x'],
obj['y'],
obj['a'],
obj['b'],
obj['theta'],
r=2)
tmpmask3d[0, :, :] = tmpmask[:, :]
srcmask = srcmask + tmpmask3d * nbj
if (spectra):
savename = "{}/id{}.fits".format(outspec, nbj)
utl.cube2spec(cube,
obj['x'],
obj['y'],
None,
write=savename,
shape='mask',
helio=helio,
mask=tmpmask3d,
tovac=True)
#go to next
nbj = nbj + 1
if (check):
print 'Dumping source mask...'
hdumain = fits.PrimaryHDU(srcmask, header=header)
hdubk = fits.ImageHDU(srcmask)
hdulist = fits.HDUList([hdumain, hdubk])
hdulist.writeto(output + "/source.fits", clobber=True)
print 'Dumping segmentation map'
hdumain = fits.PrimaryHDU(segmap, header=header)
hdubk = fits.ImageHDU(segmap)
hdulist = fits.HDUList([hdumain, hdubk])
hdulist.writeto(output + "/segmap.fits", clobber=True)
#write source catalogue
print 'Writing catalogue..'
cols = fits.ColDefs(objects)
tbhdu = fits.BinTableHDU.from_columns(cols)
tbhdu.writeto(output + '/catalogue.fits', clobber=True)
print 'All done'
return objects
def fixandsky_firstpass(cube, pixtab, noclobber, skymask=None):
"""
Take a cube and pixel table and fix the cube, then skysub and produce white image,
using CubEx utils
is skymask is set, mask regions when computing normalisation
"""
import os
import subprocess
import numpy as np
from astropy.io import fits
from mypython.fits import pyregmask as pmk
import matplotlib.pyplot as plt
#make some intermediate names
fixed = cube.split('.fits')[0] + "_fix.fits"
skysub = cube.split('.fits')[0] + "_skysub.fits"
white = cube.split('.fits')[0] + "_white.fits"
sharpmsk = cube.split('.fits')[0] + "_sharpmask.fits"
#if told to mask sky do it.. otherwise leave image empty
cb = fits.open(cube)
nx = cb[1].header['NAXIS1']
ny = cb[1].header['NAXIS2']
sharpmask = np.zeros((ny, nx))
if (skymask):
#construct the sky region mask
mysky = pmk.PyMask(nx, ny, "../../" + skymask, header=cb[1].header)
for ii in range(mysky.nreg):
mysky.fillmask(ii)
sharpmask = sharpmask + mysky.mask
#write mask
hdu = fits.PrimaryHDU(sharpmask)
hdulist = fits.HDUList([hdu])
hdulist.writeto(sharpmsk, clobber=True)
cb.close()
#now fix the cube
if ((os.path.isfile(fixed)) & (noclobber)):
print "Cube {0} already fixed".format(cube)
else:
print 'Cubefix ', cube
subprocess.call(
["CubeFix", "-cube", cube, "-pixtable", pixtab, "-out", fixed])
#now run cube skysub
if ((os.path.isfile(skysub)) & (noclobber)):
print "Cube {0} already skysub".format(fixed)
else:
print 'Sky sub ', fixed
subprocess.call([
"CubeSharp", "-cube", fixed, "-out", skysub, "-sourcemask",
sharpmsk, "-lcheck", ".false."
])
#create a white image
if ((os.path.isfile(white)) & (noclobber)):
print "White image for cube {0} already exists".format(skysub)
else:
print 'Create white image for ', skysub
subprocess.call(["Cube2Im", "-cube", skysub, "-out", white])
def fixandsky_secondpass(cube,pixtab,noclobber,highsn=None,skymask=None,exthsnmask=None,version='1.6'):
"""
Similar to first pass, but operates on cubes that have been realigned and uses masks
as appropriate
If highsn cube is provided, use it to mask the sources and for skysubtraction
"""
import os
import subprocess
import numpy as np
from astropy.io import fits
from mypython.fits import pyregmask as pmk
#define the step to catch error
if('1.8' in version):
errorstep='4'
else:
errorstep='3'
if(highsn):
#prepare final names
fixed=cube.split('.fits')[0]+"_fixhsn.fits"
skysub=cube.split('.fits')[0]+"_skysubhsn.fits"
white=cube.split('.fits')[0]+"_whitehsn.fits"
sharpmsk=cube.split('.fits')[0]+"_sharpmaskhsn.fits"
else:
#prepare intermediate names
fixed=cube.split('.fits')[0]+"_fix2.fits"
skysub=cube.split('.fits')[0]+"_skysub2.fits"
white=cube.split('.fits')[0]+"_white2.fits"
sharpmsk=cube.split('.fits')[0]+"_sharpmask2.fits"
#now fix the cube using masks
if ((os.path.isfile(fixed)) & (noclobber)):
print("Cube {0} already fixed".format(cube))
else:
print('Create source mask...')
if(highsn):
#The final mask will always have the same name and will overwrite existing file
mask_source=cube.split('.fits')[0]+"_whitedeep.Objects_Id.fits"
if exthsnmask:
print('Using external HSN mask...')
subprocess.call(["cp", exthsnmask, mask_source])
else:
print('Using high SN cube...')
#create source mask from deep exposure
white_source=cube.split('.fits')[0]+"_whitedeep.fits"
subprocess.call(["Cube2Im","-cube",highsn,"-out",white_source])
subprocess.call(["CubEx",white_source,'-MultiExt','.false.','-ApplyFilter','.true.','-ApplyFilterVar','.true.','-FilterXYRad','1','-SN_Threshold','7','-MinNSpax','5'])
else:
print('Using white image from previous loop')
#create source mask from previous step
mask_source=cube.split('.fits')[0]+"_white.Objects_Id.fits"
white_source=cube.split('.fits')[0]+"_white.fits"
subprocess.call(["CubEx",white_source,'-MultiExt','.false.','-SN_Threshold','4.5','-RescaleVar','.true.'])
print('Cubefix ', cube)
subprocess.call(["CubeFix","-cube", cube,"-pixtable", pixtab,"-out", fixed,"-sourcemask",mask_source])
#catch werid cases in which cubefix crashes
if not os.path.isfile(fixed):
#try again with last step only
print('Redo fix with last step only')
subprocess.call(["CubeFix","-cube", cube,"-pixtable", pixtab,"-out", fixed,"-sourcemask",mask_source,"-step",errorstep])
#At this step, check out cubeAdd2Mask if want to fix edges or weird ifus/slices
#if told to mask a particular sky region do it.. otherwise leave image empty
cb=fits.open(cube)
nx=cb[1].header['NAXIS1']
ny=cb[1].header['NAXIS2']
sharpmask=np.zeros((ny,nx))
if(skymask):
#construct the sky region mask
mysky=pmk.PyMask(nx,ny,"../../"+skymask,header=cb[1].header)
for ii in range(mysky.nreg):
mysky.fillmask(ii)
sharpmask=sharpmask+mysky.mask
cb.close()
#inject src mask in sharpmask
srcmk=fits.open(mask_source)
sharpmask=sharpmask+srcmk[0].data
#write mask
hdu = fits.PrimaryHDU(sharpmask)
hdulist = fits.HDUList([hdu])
hdulist.writeto(sharpmsk,overwrite=True)
#now run cube skysub
if ((os.path.isfile(skysub)) & (noclobber)):
print("Cube {0} already skysub".format(fixed))
else:
print('Sky sub ', fixed)
if(highsn):
#now few more options to control sky sub
subprocess.call(["CubeSharp","-cube",fixed,"-out",skysub,"-sourcemask",sharpmsk,"-hsncube",highsn,"-lsig","5"])
else:
subprocess.call(["CubeSharp","-cube",fixed,"-out",skysub,"-sourcemask",sharpmsk])
#create a white image
if ((os.path.isfile(white)) & (noclobber)):
print("White image for cube {0} already exists".format(skysub))
else:
print('Create white image for ', skysub)
subprocess.call(["Cube2Im","-cube",skysub,"-out",white])
def selfcalibrate(listob,
deepwhite,
refpath='esocombine',
nproc=24,
extmask=None,
extmaskonly=False):
"""
Loop over each OB and performs self-calibration after masking sources as
listob -> OBs to process
deepwhite -> the best white image available to mask sources
refpath -> where to find a white image to be used as reference wcs grid
nproc -> number of processors
extmask -> ds9 region file in image coordinates defining the regions to be masked
extmaskonly -> if true ose only the mask supplied externally, if false merge sextractor and extmasks
"""
import os
import glob
import subprocess
import shutil
from astropy.io import fits
import muse_utils as mut
import numpy as np
import sep
from mpdaf.drs import PixTable
from mypython.fits import pyregmask as pmk
#grab top dir
topdir = os.getcwd()
#now loop over each folder and make the final sky-subtracted cubes
for ob in listob:
#change dir
os.chdir(ob + '/Proc/MPDAF')
print("Processing {} for self-calibration".format(ob))
#make source mask
srcmask = 'selfcalib_mask.fits'
if (os.path.isfile(srcmask)):
print("Source mask already exists!")
else:
print("Create source mask for selfcalibration")
#open the ifu mask to create a good mask
deepdata = fits.open("../../../" + deepwhite)
#now flag the sources (allow cubex/eso images)
try:
image = deepdata[0].data.byteswap().newbyteorder()
datheader = deepdata[0].header
except:
image = deepdata[1].data.byteswap().newbyteorder()
datheader = deepdata[1].header
bkg = sep.Background(image)
bkg.subfrom(image)
obj, segmap = sep.extract(image,
5. * bkg.globalrms,
minarea=6,
segmentation_map=True)
segmap[np.where(segmap > 0)] = 1
#write source mask to disk
hdu = fits.PrimaryHDU(segmap, header=datheader)
hdu.writeto(srcmask, overwrite=True)
if (extmask):
extfitsmask = 'ext_mask.fits'
#Read geometry from automatic mask
srchdu = fits.open(srcmask)
srchead = srchdu[0].header
srcmsk = srchdu[0].data
extmsk = np.zeros_like(srcmsk)
nx = srchead['NAXIS1']
ny = srchead['NAXIS2']
#construct the sky region mask
mysky = pmk.PyMask(nx,
ny,
"../../../" + extmask,
header=srchdu[0].header)
for ii in range(mysky.nreg):
mysky.fillmask(ii)
extmsk = extmsk + mysky.mask
outhdu = fits.PrimaryHDU(extmsk, header=srchead)
outhdu.writeto(extfitsmask, overwrite=True)
srchdu.close()
if (extmask and not extmaskonly):
namecombmask = 'extsrc_mask.fits'
finalmsk = extmsk + srcmsk
finalmsk[finalmsk > 0] = 1
outhdu = fits.PrimaryHDU(finalmsk, header=srchead)
outhdu.writeto(namecombmask, overwrite=True)
finalmask = namecombmask
elif (extmask and extmaskonly):
finalmask = extfitsmask
else:
finalmask = srcmask
#now loop over exposures and apply self calibration
scils = glob.glob("../Basic/OBJECT_RED_0*.fits*")
nsci = len(scils)
#loop on exposures and apply self calibration
for exp in range(nsci):
pixselfcal = "PIXTABLE_REDUCED_RESAMPLED_EXP{0:d}_fix.fits".format(
exp + 1)
if not os.path.isfile(pixselfcal):
print("Apply self-calibration to {}".format(pixselfcal))
#open reduced pixel table
pix = PixTable(
"PIXTABLE_REDUCED_RESAMPLED_EXP{0:d}.fits".format(exp + 1))
#create mask
maskpix = pix.mask_column(finalmask)
maskpix.write(
"PIXTABLE_REDUCED_RESAMPLED_EXP{0:d}_mask.fits".format(
exp + 1))
#selfcalibrate
autocalib = pix.selfcalibrate(pixmask=maskpix)
#write to disk
autocalib.write(
"PIXTABLE_REDUCED_RESAMPLED_EXP{0:d}_autocalib.fits".
format(exp + 1))
pix.write(pixselfcal)
else:
print("Self-calibration for {} already done! Skip...".format(
pixselfcal))
cubeselfcal = "DATACUBE_RESAMPLED_EXP{0:d}_fix.fits".format(exp +
1)
imageselfcal = "IMAGE_RESAMPLED_EXP{0:d}_fix.fits".format(exp + 1)
if not os.path.isfile(cubeselfcal):
print('Reconstruct cube {}'.format(cubeselfcal))
#now reconstruct cube and white image on right reference frame
#handle sof file
sof_name = "../../Script/scipost_mpdaf_self{0:d}.sof".format(
exp + 1)
sofedit = open(sof_name, 'w')
sofedit.write(
'../../../{}/DATACUBE_FINAL.fits OUTPUT_WCS\n'.format(
refpath))
sofedit.write(
"PIXTABLE_REDUCED_RESAMPLED_EXP{0:d}_fix.fits PIXTABLE_OBJECT\n"
.format(exp + 1))
sofedit.close()
#now run script
scr = open(
"../../Script/make_scipost_mpdaf_self{0:d}.sh".format(exp +
1),
"w")
scr.write("OMP_NUM_THREADS={0:d}\n".format(nproc))
scr.write(
'esorex --log-file=scipost_mpdaf_self{0:d}.log muse_scipost_make_cube ../../Script/scipost_mpdaf_self{0:d}.sof'
.format(exp + 1))
scr.close()
#Run pipeline
subprocess.call([
"sh",
"../../Script/make_scipost_mpdaf_self{0:d}.sh".format(exp +
1)
])
subprocess.call(["mv", "DATACUBE_FINAL.fits", cubeselfcal])
subprocess.call(["mv", "IMAGE_FOV_0001.fits", imageselfcal])
else:
print('Cube {} already exists! Skip...'.format(cubeselfcal))
#back to top
os.chdir(topdir)
def zapskysub(listob, extmask=None, extmaskonly=False):
"""
Loop over each OB and performs zap sky subtraction
listob -> OBs to process
"""
import os
import glob
import subprocess
import shutil
from astropy.io import fits
import muse_utils as mut
import numpy as np
import sep
import zap
from mypython.fits import pyregmask as pmk
#grab top dir
topdir = os.getcwd()
#now loop over each folder and make the final sky-subtracted cubes
for ob in listob:
#change dir
os.chdir(ob + '/Proc/MPDAF')
#use source mask already available (should we assume it is always available?)
srcmask = 'selfcalib_mask.fits'
if (extmask):
extfitsmask = 'ext_mask.fits'
srchdu = fits.open(srcmask)
srcmsk = srchdu[0].data
extmsk = np.zeros_like(srcmsk)
nx = srchdu[0].header['NAXIS1']
ny = srchdu[0].header['NAXIS2']
#construct the sky region mask
mysky = pmk.PyMask(nx,
ny,
"../../../" + extmask,
header=srchdu[0].header)
for ii in range(mysky.nreg):
mysky.fillmask(ii)
extmsk = extmsk + mysky.mask
outhdu = fits.PrimaryHDU(extmsk)
outhdu.writeto(extfitsmask, overwrite=True)
srchdu.close()
if (extmask and extmaskonly):
namecombmask = 'extsrc_mask.fits'
finalmsk = extmsk + srcmsk
finalmsk[finalmsk > 0] = 1
outhdu = fits.PrimaryHDU(finalmsk)
outhdu.writeto(namecombmask, overwrite=True)
finalmask = namecombmask
elif (extmask and not extmaskonly):
finalmask = extfitsmask
else:
finalmask = srcmask
#now loop over exposures and apply self calibration
scils = glob.glob("../Basic/OBJECT_RED_0*.fits*")
nsci = len(scils)
print("Processing {} with ZAP".format(ob))
#loop on exposures and apply self calibration
for exp in range(nsci):
#these are the self-calibrated data
cubeselfcal = "DATACUBE_RESAMPLED_EXP{0:d}_fix.fits".format(exp +
1)
imageselfcal = "IMAGE_RESAMPLED_EXP{0:d}_fix.fits".format(exp + 1)
#these are the sky subtracted products
cubezap = "DATACUBE_RESAMPLED_EXP{0:d}_zap.fits".format(exp + 1)
imagezap = "IMAGE_RESAMPLED_EXP{0:d}_zap.fits".format(exp + 1)
if not os.path.isfile(cubezap):
print('Reconstruct cube {} with ZAP'.format(cubezap))
#check if first 50 slices are all nan (typically from mixing E and N modes)
currcube = fits.open(cubeselfcal)
slice50 = currcube[1].data[0:50, :, :]
goodpix = np.count_nonzero(np.isfinite(slice50))
if (goodpix < 1):
#scan layers to find first with value
endslice = 1
while (goodpix < 1):
slice50 = currcube[1].data[0:endslice, :, :]
goodpix = np.count_nonzero(np.isfinite(slice50))
endslice = endslice + 1
#find start index for good data
endslice = endslice - 2
#write tmp trimmed cube
hdu1 = fits.PrimaryHDU([], header=currcube[0].header)
hdu2 = fits.ImageHDU(currcube[1].data[endslice:, :, :],
header=currcube[1].header)
hdu3 = fits.ImageHDU(currcube[2].data[endslice:, :, :],
header=currcube[2].header)
#update wave solution
hdu2.header['CRVAL3'] = currcube[1].header[
'CRVAL3'] + endslice * currcube[1].header['CD3_3']
hdu3.header['CRVAL3'] = currcube[2].header[
'CRVAL3'] + endslice * currcube[2].header['CD3_3']
#write to new file
hdul = fits.HDUList([hdu1, hdu2, hdu3])
hdul.writeto('trimmed_' + cubeselfcal, overwrite=True)
currcube.close()
#now run zap
zap.process('trimmed_' + cubeselfcal,
outcubefits='trimmed_' + cubezap,
clean=True,
mask=finalmask)
#make copy of original cube
shutil.copy(cubeselfcal, cubezap)
#now update
longcube = fits.open(cubezap, mode='update')
shortcube = fits.open('trimmed_' + cubezap)
longcube[1].data[endslice:, :, :] = shortcube[1].data
longcube[2].data[endslice:, :, :] = shortcube[2].data
longcube.flush()
longcube.close()
shortcube.close()
else:
#proceed with current data
currcube.close()
zap.process(cubeselfcal,
outcubefits=cubezap,
clean=True,
mask=finalmask)
#create white image from zap cube
cube = fits.open(cubezap)
#define geometry
nwave = cube[1].header["NAXIS3"]
nx = cube[1].header["NAXIS1"]
ny = cube[1].header["NAXIS2"]
print('Creating final white image from ZAP')
white_new = np.zeros((ny, nx))
for xx in range(nx):
for yy in range(ny):
white_new[yy, xx] = np.nansum(cube[1].data[:, yy,
xx]) / nwave
#save projected image
hdu1 = fits.PrimaryHDU([])
hdu2 = fits.ImageHDU(white_new)
hdu2.header = cube[1].header
hdulist = fits.HDUList([hdu1, hdu2])
hdulist.writeto(imagezap, overwrite=True)
else:
print('Cube {} already exists! Skip...'.format(cubezap))
#back to top
os.chdir(topdir)
def coaddcubes(listob, nclip=2.5):
"""
Loop over each OB and make final combined (mean, median) cubes
listob -> OBs to process
nclip -> threshold for sigmaclipping
"""
import os
import glob
import subprocess
import shutil
from astropy.io import fits
import muse_utils as mut
import numpy as np
import matplotlib.pyplot as plt
#names for median/mean output
cubemed = "mpdafcombine/COMBINED_CUBE_MED_FINAL.fits"
imagemed = "mpdafcombine/COMBINED_IMAGE_MED_FINAL.fits"
cubemean = "mpdafcombine/COMBINED_CUBE_FINAL.fits"
imagemean = "mpdafcombine/COMBINED_IMAGE_FINAL.fits"
if ((not os.path.isfile(cubemed)) | (not os.path.isfile(cubemean))):
print('Compute mean/median coadd {} {}'.format(cubemed, cubemean))
#first collect all relevant exposures
allexposures = []
for ob in listob:
finalexp = glob.glob(
"{}/Proc/MPDAF/DATACUBE_RESAMPLED_EXP*_zap.fits".format(ob))
allexposures += finalexp
#loop over exposures for combine
for i, exp in enumerate(allexposures):
print('Ingesting data {}'.format(exp))
#open exposure
data = fits.open(exp)
#operations to do only once
if (i == 0):
nw, nx, ny = data[1].data.shape
nexp = len(allexposures)
datashape = (nexp, nw, nx, ny)
alldata = np.ma.zeros(datashape)
alldata.mask = np.full(datashape, False)
headermain = data[0].header
headerext = data[1].header
headervar = data[2].header
#store data
alldata.data[i] = data[1].data
#catch nans in this exposure
alldata.mask[i] = np.logical_not(np.isfinite(data[1].data))
data.close()
#now handle mask for this exposure if available in cubex or mpdaf region
mpdafmask = exp.split("DATACUBE")[0] + 'IMAGE' + exp.split(
"DATACUBE")[1]
mpdafreg = mpdafmask.split('fits')[0] + 'reg'
cubexmask = exp.split("_RESAMPLED")
cubexmask = cubexmask[0] + "_FINAL_RESAMPLED" + cubexmask[1]
cubexmask = cubexmask.replace('MPDAF', 'Cubex')
cubexmask1 = cubexmask.replace('_zap.fits',
'_fixhsn_SliceEdgeMask_wreg.fits')
cubexmask2 = cubexmask.replace('_zap.fits',
'_fixhsn_SliceEdgeMask.fits')
#handle mpdaf first
if os.path.isfile(mpdafreg):
#now fill using region
Mask = msk.PyMask(ny, nx, mpdafreg)
for ii in range(Mask.nreg):
Mask.fillmask(ii)
if (ii == 0):
slicemask = Mask.mask
else:
slicemask += Mask.mask
slicecube = np.tile(slicemask, (nw, 1, 1))
alldata.mask[i] = slicecube
cubexmask = None
#else, look at cubex and wor as selector
elif os.path.isfile(cubexmask1):
cubexmask = cubexmask1
elif os.path.isfile(cubexmask2):
cubexmask = cubexmask2
else:
cubexmask = None
#fill in cubex mask if available
if (cubexmask):
#mask edges
fitsmask = fits.open(cubexmask)
slicemask = np.full(fitsmask[0].data.shape, False)
masked = np.where(fitsmask[0].data < 1)
slicemask[masked] = True
#grow cube
slicecube = np.tile(slicemask, (nw, 1, 1))
alldata.mask[i] = slicecube
fitsmask.close()
#sanitise output
alldata = np.ma.masked_invalid(alldata)
#iterative sigmaclipping
for niter in range(3):
#now catch significant outliers
print(
'Catching outliers with sigma clipping; loop {}'.format(niter +
1))
stdevtmp = alldata.std(axis=0)
medtmp = np.ma.median(alldata, axis=0)
#loop over exposures
for i, exp in enumerate(allexposures):
print('Masking outliers in {}'.format(exp))
alldata[i] = np.ma.masked_where(
abs(alldata[i] - medtmp) >= nclip * stdevtmp, alldata[i])
#save exposure time map
print('Compute exposure map')
expcube = alldata.count(axis=0)
expmap = np.sum(expcube, axis=0) / nw
#at last perform median combine
print('Computing median')
medcube = np.ma.median(alldata, axis=0)
#at last perform mean combine
print('Computing mean')
meancube = alldata.mean(axis=0)
#now ingest variance
for i, exp in enumerate(allexposures):
print('Ingesting variance {}'.format(exp))
#open exposure
data = fits.open(exp)
#store data
alldata.data[i] = data[2].data
data.close()
print('Computing variance')
varcube = alldata.sum(axis=0) / expcube / expcube
#save exposure time map
hdu1 = fits.PrimaryHDU([])
hdu2 = fits.ImageHDU(expmap)
hdu2.header = headerext
hdulist = fits.HDUList([hdu1, hdu2])
hdulist.writeto("mpdafcombine/FINAL_COADD_EXPOSUREMAP.fits",
overwrite=True)
#save output median
hdu = fits.PrimaryHDU([])
hdu1 = fits.ImageHDU(medcube.data)
hdu2 = fits.ImageHDU(varcube.data)
hdulist = fits.HDUList([hdu, hdu1, hdu2])
hdulist[0].header = headermain
hdulist[1].header = headerext
hdulist[2].header = headervar
hdulist.writeto(cubemed, overwrite=True)
#now make white image
print('Creating final white image from median cube')
white_new = np.zeros((nx, ny))
for xx in range(nx):
for yy in range(ny):
#skip couple of channells for cosmetics
white_new[xx, yy] = np.nansum(medcube.data[2:-3, xx,
yy]) / (nw - 4)
#save projected image
hdu1 = fits.PrimaryHDU([])
hdu2 = fits.ImageHDU(white_new)
hdu2.header = headerext
hdulist = fits.HDUList([hdu1, hdu2])
hdulist.writeto(imagemed, overwrite=True)
#save output mean
hdu = fits.PrimaryHDU([])
hdu1 = fits.ImageHDU(meancube.data)
hdu2 = fits.ImageHDU(varcube.data)
hdulist = fits.HDUList([hdu, hdu1, hdu2])
hdulist[0].header = headermain
hdulist[1].header = headerext
hdulist[2].header = headervar
hdulist.writeto(cubemean, overwrite=True)
#now make white image
print('Creating final white image from mean cube')
white_new = np.zeros((nx, ny))
for xx in range(nx):
for yy in range(ny):
#skip couple of channells for cosmetics
white_new[xx, yy] = np.nansum(meancube.data[2:-3, xx,
yy]) / (nw - 4)
#save projected image
hdu1 = fits.PrimaryHDU([])
hdu2 = fits.ImageHDU(white_new)
hdu2.header = headerext
hdulist = fits.HDUList([hdu1, hdu2])
hdulist.writeto(imagemean, overwrite=True)
else:
print('Median and mean coadd {} already exists!'.format(
cubemed, cubemean))
def zapskysub(listob, extmask=None, extmaskonly=False):
"""
Loop over each OB and performs zap sky subtraction
listob -> OBs to process
"""
import os
import glob
import subprocess
import shutil
from astropy.io import fits
import muse_utils as mut
import numpy as np
import sep
import zap
from mypython.fits import pyregmask as pmk
#grab top dir
topdir = os.getcwd()
#now loop over each folder and make the final sky-subtracted cubes
for ob in listob:
#change dir
os.chdir(ob + '/Proc/MPDAF')
#use source mask already available (should we assume it is always available?)
srcmask = 'selfcalib_mask.fits'
if (extmask):
extfitsmask = 'ext_mask.fits'
srchdu = fits.open(srcmask)
srcmsk = srchdu[0].data
extmsk = np.zeros_like(srcmsk)
nx = srchdu[0].header['NAXIS1']
ny = srchdu[0].header['NAXIS2']
#construct the sky region mask
mysky = pmk.PyMask(nx,
ny,
"../../../" + extmask,
header=srchdu[0].header)
for ii in range(mysky.nreg):
mysky.fillmask(ii)
extmsk = extmsk + mysky.mask
outhdu = fits.PrimaryHDU(extmsk)
outhdu.writeto(extfitsmask, overwrite=True)
srchdu.close()
if (extmask and extmaskonly):
namecombmask = 'extsrc_mask.fits'
finalmsk = extmsk + srcmsk
finalmsk[finalmsk > 0] = 1
outhdu = fits.PrimaryHDU(finalmsk)
outhdu.writeto(namecombmask, overwrite=True)
finalmask = namecombmask
elif (extmask and not extmaskonly):
finalmask = extfitsmask
else:
finalmask = srcmask
#now loop over exposures and apply self calibration
scils = glob.glob("../Basic/OBJECT_RED_0*.fits*")
nsci = len(scils)
print("Processing {} with ZAP".format(ob))
#loop on exposures and apply self calibration
for exp in range(nsci):
#these are the self-calibrated data
cubeselfcal = "DATACUBE_RESAMPLED_EXP{0:d}_fix.fits".format(exp +
1)
imageselfcal = "IMAGE_RESAMPLED_EXP{0:d}_fix.fits".format(exp + 1)
#these are the sky subtracted products
cubezap = "DATACUBE_RESAMPLED_EXP{0:d}_zap.fits".format(exp + 1)
imagezap = "IMAGE_RESAMPLED_EXP{0:d}_zap.fits".format(exp + 1)
if not os.path.isfile(cubezap):
print('Reconstruct cube {} with ZAP'.format(cubezap))
zap.process(cubeselfcal,
outcubefits=cubezap,
clean=True,
mask=finalmask)
#create white image from zap cube
cube = fits.open(cubezap)
#define geometry
nwave = cube[1].header["NAXIS3"]
nx = cube[1].header["NAXIS1"]
ny = cube[1].header["NAXIS2"]
print('Creating final white image from ZAP')
white_new = np.zeros((ny, nx))
for xx in range(nx):
for yy in range(ny):
white_new[yy, xx] = np.nansum(cube[1].data[:, yy,
xx]) / nwave
#save projected image
hdu1 = fits.PrimaryHDU([])
hdu2 = fits.ImageHDU(white_new)
hdu2.header = cube[1].header
hdulist = fits.HDUList([hdu1, hdu2])
hdulist.writeto(imagezap, overwrite=True)
else:
print('Cube {} already exists! Skip...'.format(cubezap))
#back to top
os.chdir(topdir)
