def test_select_sky(numexpr, shape, pixfile):
pix = PixTable(pixfile)
with toggle_numexpr(numexpr):
x, y = pix.get_pos_sky()
cx = (x.min() + x.max()) / 2
cy = (y.min() + y.max()) / 2
pix2 = pix.extract(sky=(cy, cx, 12, shape))
assert pix2.nrows < pix.nrows
def test_reconstruct(pixfile):
pix = PixTable(pixfile).extract(ifu=1)
im = pix.reconstruct_det_image()
assert im.shape == (1101, 1920)
im = pix.reconstruct_det_waveimage()
assert im.shape == (1101, 1920)
def test_select(numexpr, pixfile):
pix = PixTable(pixfile)
with toggle_numexpr(numexpr):
pix2 = pix.extract(xpix=[(1000, 2000)], ypix=[(1000, 2000)])
origin = pix2.get_origin()
xpix = pix2.origin2xpix(origin)
ypix = pix2.origin2ypix(origin)
assert xpix.min() >= 1000
assert xpix.max() <= 2000
assert ypix.min() >= 1000
assert ypix.max() <= 2000
def test_write(self):
tmpdir = tempfile.mkdtemp(suffix='.mpdaf-test-pixtable')
out = join(tmpdir, 'PIX.fits')
self.pix.write(out)
pix1 = PixTable(out)
out = join(tmpdir, 'PIX2.fits')
self.pix.write(out, save_as_ima=False)
pix2 = PixTable(out)
for p in (pix1, pix2):
self.assertEqual(self.pix.nrows, p.nrows)
for name in ('xpos', 'ypos', 'data', 'dq', 'stat', 'origin'):
assert_allclose(
getattr(p, 'get_' + name)(), getattr(self, name))
pix1.hdulist.close()
pix2.hdulist.close()
def setUp(self):
np.random.seed(42)
self.xpos = np.linspace(1, 10, NROWS)
self.ypos = np.linspace(2, 6, NROWS)
self.lbda = np.linspace(5000, 8000, NROWS)
self.data = np.linspace(0, 100, NROWS)
self.dq = np.random.randint(0, 2, NROWS)
self.stat = np.linspace(0, 1, NROWS)
# generate origin column
np.random.seed(42)
self.aifu = np.random.randint(1, 25, NROWS)
self.aslice = np.random.randint(1, 49, NROWS)
self.ax = np.random.randint(1, 4112, NROWS)
self.ay = np.random.randint(1, 4112, NROWS)
self.aoffset = self.ax // 90 * 90
# Make sure we have at least one ifu=1 value to avoid random failures
self.aifu[0] = 1
self.origin = (((self.ax - self.aoffset) << MUSE_ORIGIN_SHIFT_XSLICE) |
(self.ay << MUSE_ORIGIN_SHIFT_YPIX) |
(self.aifu << MUSE_ORIGIN_SHIFT_IFU) | self.aslice)
prihdu = fits.PrimaryHDU()
prihdu.header['author'] = ('MPDAF', 'origin of the file')
prihdu.header['RA'] = 0.0
prihdu.header['DEC'] = 0.0
prihdu.header['HIERARCH ESO DRS MUSE PIXTABLE WCS'] = \
'projected (intermediate)'
self.pix = PixTable(None,
xpos=self.xpos,
ypos=self.ypos,
lbda=self.lbda,
data=self.data,
dq=self.dq,
stat=self.stat,
origin=self.origin,
primary_header=prihdu.header)
self.file = io.BytesIO()
shape = (NROWS, 1)
hdu = fits.HDUList([
prihdu,
fits.ImageHDU(name='xpos', data=self.xpos.reshape(shape)),
fits.ImageHDU(name='ypos', data=self.ypos.reshape(shape)),
fits.ImageHDU(name='lambda', data=self.lbda.reshape(shape)),
fits.ImageHDU(name='data', data=self.data.reshape(shape)),
fits.ImageHDU(name='dq', data=self.dq.reshape(shape)),
fits.ImageHDU(name='stat', data=self.stat.reshape(shape)),
fits.ImageHDU(name='origin', data=self.origin.reshape(shape)),
])
hdu[1].header['BUNIT'] = self.pix.wcs.to_string('fits')
hdu[2].header['BUNIT'] = self.pix.wcs.to_string('fits')
hdu[3].header['BUNIT'] = self.pix.wave.to_string('fits')
hdu[4].header['BUNIT'] = self.pix.unit_data.to_string('fits')
hdu[6].header['BUNIT'] = (self.pix.unit_data**2).to_string('fits')
hdu.writeto(self.file)
self.file.seek(0)
self.pix2 = PixTable(self.file)
self.file.seek(0)
class TestBasicPixTable(unittest.TestCase):
@classmethod
def setUp(self):
np.random.seed(42)
self.xpos = np.linspace(1, 10, NROWS)
self.ypos = np.linspace(2, 6, NROWS)
self.lbda = np.linspace(5000, 8000, NROWS)
self.data = np.linspace(0, 100, NROWS)
self.dq = np.random.randint(0, 2, NROWS)
self.stat = np.linspace(0, 1, NROWS)
# generate origin column
np.random.seed(42)
self.aifu = np.random.randint(1, 25, NROWS)
self.aslice = np.random.randint(1, 49, NROWS)
self.ax = np.random.randint(1, 4112, NROWS)
self.ay = np.random.randint(1, 4112, NROWS)
self.aoffset = self.ax // 90 * 90
# Make sure we have at least one ifu=1 value to avoid random failures
self.aifu[0] = 1
self.origin = (((self.ax - self.aoffset) << MUSE_ORIGIN_SHIFT_XSLICE) |
(self.ay << MUSE_ORIGIN_SHIFT_YPIX) |
(self.aifu << MUSE_ORIGIN_SHIFT_IFU) | self.aslice)
prihdu = fits.PrimaryHDU()
prihdu.header['author'] = ('MPDAF', 'origin of the file')
prihdu.header['RA'] = 0.0
prihdu.header['DEC'] = 0.0
prihdu.header['HIERARCH ESO DRS MUSE PIXTABLE WCS'] = \
'projected (intermediate)'
self.pix = PixTable(None,
xpos=self.xpos,
ypos=self.ypos,
lbda=self.lbda,
data=self.data,
dq=self.dq,
stat=self.stat,
origin=self.origin,
primary_header=prihdu.header)
self.file = io.BytesIO()
shape = (NROWS, 1)
hdu = fits.HDUList([
prihdu,
fits.ImageHDU(name='xpos', data=self.xpos.reshape(shape)),
fits.ImageHDU(name='ypos', data=self.ypos.reshape(shape)),
fits.ImageHDU(name='lambda', data=self.lbda.reshape(shape)),
fits.ImageHDU(name='data', data=self.data.reshape(shape)),
fits.ImageHDU(name='dq', data=self.dq.reshape(shape)),
fits.ImageHDU(name='stat', data=self.stat.reshape(shape)),
fits.ImageHDU(name='origin', data=self.origin.reshape(shape)),
])
hdu[1].header['BUNIT'] = self.pix.wcs.to_string('fits')
hdu[2].header['BUNIT'] = self.pix.wcs.to_string('fits')
hdu[3].header['BUNIT'] = self.pix.wave.to_string('fits')
hdu[4].header['BUNIT'] = self.pix.unit_data.to_string('fits')
hdu[6].header['BUNIT'] = (self.pix.unit_data**2).to_string('fits')
hdu.writeto(self.file)
self.file.seek(0)
self.pix2 = PixTable(self.file)
self.file.seek(0)
def test_empty_pixtable(self):
pix = PixTable(None)
self.assertEqual(pix.nrows, 0)
self.assertEqual(pix.extract(), None)
self.assertEqual(pix.get_data(), None)
def test_header(self):
assert self.pix.fluxcal is False
assert self.pix.skysub is False
assert self.pix.projection == 'projected'
pix2 = self.pix.copy()
pix2.primary_header['HIERARCH ESO DRS MUSE PIXTABLE FLUXCAL'] = True
assert pix2.fluxcal is True
def test_getters(self):
self.assertEqual(NROWS, self.pix.nrows)
for name in ('xpos', 'ypos', 'data', 'dq', 'stat', 'origin'):
assert_array_equal(
getattr(self.pix, 'get_' + name)(), getattr(self, name))
assert_array_equal(
getattr(self.pix2, 'get_' + name)(), getattr(self, name))
def test_get_lambda(self):
assert_array_equal(self.lbda, self.pix.get_lambda())
assert_array_equal(self.lbda * u.angstrom.to(u.nm),
self.pix.get_lambda(unit=u.nm))
ksel = np.where(self.lbda > 6000)
assert_array_equal(self.lbda[ksel], self.pix.get_lambda(ksel))
ksel = (self.lbda > 6000)
assert_array_equal(self.lbda[ksel], self.pix.get_lambda(ksel))
assert_array_equal(self.lbda, self.pix2.get_lambda())
ksel = np.where(self.lbda > 6000)
assert_array_equal(self.lbda[ksel], self.pix2.get_lambda(ksel))
ksel = (self.lbda > 6000)
assert_array_equal(self.lbda[ksel], self.pix2.get_lambda(ksel))
def test_get_xypos(self):
assert_array_equal(self.xpos, self.pix2.get_xpos())
assert_array_equal(self.xpos, self.pix.get_xpos(unit=u.pix))
assert_array_equal(self.ypos, self.pix2.get_ypos())
assert_array_equal(self.ypos, self.pix.get_ypos(unit=u.pix))
def test_get_data(self):
assert_array_equal(self.data, self.pix2.get_data())
assert_array_equal(self.data, self.pix.get_data(unit=u.count))
assert_array_equal(self.stat, self.pix2.get_stat())
assert_array_equal(self.stat, self.pix.get_stat(unit=u.count**2))
def test_get_row(self):
assert self.pix.get_row(0) == {
'xpos': 1,
'ypos': 2,
'lbda': 5000,
'data': 0,
'stat': 0,
'dq': 0,
'origin': 1241548872,
}
res = self.pix.get_row([0, 1, 2])
assert len(res['xpos']) == 3
assert_array_equal(res['origin'], [1241548872, 908518680, 541389771])
def test_set_column(self):
for pixt in (self.pix, self.pix2):
pix = pixt.copy()
with self.assertRaises(AssertionError):
pix.set_xpos(list(range(5)))
for pix in (self.pix, self.pix2):
new_xpos = np.linspace(2, 3, pix.nrows)
pix.set_xpos(new_xpos)
assert_array_equal(new_xpos, pix.get_xpos())
new_ypos = np.linspace(2, 3, pix.nrows)
pix.set_ypos(new_ypos)
assert_array_equal(new_ypos, pix.get_ypos())
new_lambda = np.linspace(4000, 5000, pix.nrows)
pix.set_lambda(new_lambda)
assert_array_equal(new_lambda, pix.get_lambda())
def test_set_data(self):
for pix in (self.pix, self.pix2):
new_data = pix.get_data()
new_stat = pix.get_stat()
ksel = np.where(new_data > 50)
new_data[ksel] = 0
new_stat[ksel] = 0
pix.set_data(0, ksel=ksel)
assert_array_equal(new_data, pix.get_data())
pix.set_stat(0, ksel=ksel)
assert_array_equal(new_stat, pix.get_stat())
new_data = pix.get_data()
new_stat = pix.get_stat()
ksel = (new_data > 60)
new_data[ksel] = 1
new_stat[ksel] = 0
pix.set_data(1, ksel=ksel, unit=u.count)
assert_array_equal(new_data, pix.get_data())
pix.set_stat(1, ksel=ksel, unit=u.count**2)
assert_array_equal(new_stat, pix.get_stat())
def test_origin_conversion(self):
origin = self.pix.get_origin()
ifu = self.pix.origin2ifu(origin)
sli = self.pix.origin2slice(origin)
assert_array_equal(self.aifu, ifu)
assert_array_equal(self.aslice, sli)
assert_array_equal(self.ay, self.pix.origin2ypix(origin) + 1)
# TODO: This one needs a pixtable with a header which contains the
# offset values HIERARCH ESO DRS MUSE PIXTABLE EXP0 IFU* SLICE* XOFFSET
# assert_array_equal(self.ax,
# self.origin2xpix(origin, ifu=ifu, sli=sli))
def test_extract(self):
for numexpr in (True, False):
with toggle_numexpr(numexpr):
pix = self.pix.extract(lbda=(5000, 6000))
ksel = (self.lbda >= 5000) & (self.lbda < 6000)
assert_array_equal(self.data[ksel], pix.get_data())
pix = self.pix.extract(ifu=1)
ksel = (self.aifu == 1)
assert_array_equal(self.data[ksel], pix.get_data())
pix = self.pix.extract(sl=(1, 2, 3))
ksel = ((self.aslice == 1) | (self.aslice == 2) |
(self.aslice == 3))
assert_array_equal(self.data[ksel], pix.get_data())
def test_write(self):
tmpdir = tempfile.mkdtemp(suffix='.mpdaf-test-pixtable')
out = join(tmpdir, 'PIX.fits')
self.pix.write(out)
pix1 = PixTable(out)
out = join(tmpdir, 'PIX2.fits')
self.pix.write(out, save_as_ima=False)
pix2 = PixTable(out)
for p in (pix1, pix2):
self.assertEqual(self.pix.nrows, p.nrows)
for name in ('xpos', 'ypos', 'data', 'dq', 'stat', 'origin'):
assert_allclose(
getattr(p, 'get_' + name)(), getattr(self, name))
pix1.hdulist.close()
pix2.hdulist.close()
def test_empty_pixtable(self):
pix = PixTable(None)
self.assertEqual(pix.nrows, 0)
self.assertEqual(pix.extract(), None)
self.assertEqual(pix.get_data(), None)
def selfcalibrate(listob,
deepwhite,
refpath='esocombine',
nproc=24,
extmask=None,
extmaskonly=False):
"""
Loop over each OB and performs self-calibration after masking sources as
listob -> OBs to process
deepwhite -> the best white image available to mask sources
refpath -> where to find a white image to be used as reference wcs grid
nproc -> number of processors
extmask -> ds9 region file in image coordinates defining the regions to be masked
extmaskonly -> if true ose only the mask supplied externally, if false merge sextractor and extmasks
"""
import os
import glob
import subprocess
import shutil
from astropy.io import fits
import muse_utils as mut
import numpy as np
import sep
from mpdaf.drs import PixTable
from mypython.fits import pyregmask as pmk
#grab top dir
topdir = os.getcwd()
#now loop over each folder and make the final sky-subtracted cubes
for ob in listob:
#change dir
os.chdir(ob + '/Proc/MPDAF')
print("Processing {} for self-calibration".format(ob))
#make source mask
srcmask = 'selfcalib_mask.fits'
if (os.path.isfile(srcmask)):
print("Source mask already exists!")
else:
print("Create source mask for selfcalibration")
#open the ifu mask to create a good mask
deepdata = fits.open("../../../" + deepwhite)
#now flag the sources (allow cubex/eso images)
try:
image = deepdata[0].data.byteswap().newbyteorder()
datheader = deepdata[0].header
except:
image = deepdata[1].data.byteswap().newbyteorder()
datheader = deepdata[1].header
bkg = sep.Background(image)
bkg.subfrom(image)
obj, segmap = sep.extract(image,
5. * bkg.globalrms,
minarea=6,
segmentation_map=True)
segmap[np.where(segmap > 0)] = 1
#write source mask to disk
hdu = fits.PrimaryHDU(segmap, header=datheader)
hdu.writeto(srcmask, overwrite=True)
if (extmask):
extfitsmask = 'ext_mask.fits'
#Read geometry from automatic mask
srchdu = fits.open(srcmask)
srchead = srchdu[0].header
srcmsk = srchdu[0].data
extmsk = np.zeros_like(srcmsk)
nx = srchead['NAXIS1']
ny = srchead['NAXIS2']
#construct the sky region mask
mysky = pmk.PyMask(nx,
ny,
"../../../" + extmask,
header=srchdu[0].header)
for ii in range(mysky.nreg):
mysky.fillmask(ii)
extmsk = extmsk + mysky.mask
outhdu = fits.PrimaryHDU(extmsk, header=srchead)
outhdu.writeto(extfitsmask, overwrite=True)
srchdu.close()
if (extmask and not extmaskonly):
namecombmask = 'extsrc_mask.fits'
finalmsk = extmsk + srcmsk
finalmsk[finalmsk > 0] = 1
outhdu = fits.PrimaryHDU(finalmsk, header=srchead)
outhdu.writeto(namecombmask, overwrite=True)
finalmask = namecombmask
elif (extmask and extmaskonly):
finalmask = extfitsmask
else:
finalmask = srcmask
#now loop over exposures and apply self calibration
scils = glob.glob("../Basic/OBJECT_RED_0*.fits*")
nsci = len(scils)
#loop on exposures and apply self calibration
for exp in range(nsci):
pixselfcal = "PIXTABLE_REDUCED_RESAMPLED_EXP{0:d}_fix.fits".format(
exp + 1)
if not os.path.isfile(pixselfcal):
print("Apply self-calibration to {}".format(pixselfcal))
#open reduced pixel table
pix = PixTable(
"PIXTABLE_REDUCED_RESAMPLED_EXP{0:d}.fits".format(exp + 1))
#create mask
maskpix = pix.mask_column(finalmask)
maskpix.write(
"PIXTABLE_REDUCED_RESAMPLED_EXP{0:d}_mask.fits".format(
exp + 1))
#selfcalibrate
autocalib = pix.selfcalibrate(pixmask=maskpix)
#write to disk
autocalib.write(
"PIXTABLE_REDUCED_RESAMPLED_EXP{0:d}_autocalib.fits".
format(exp + 1))
pix.write(pixselfcal)
else:
print("Self-calibration for {} already done! Skip...".format(
pixselfcal))
cubeselfcal = "DATACUBE_RESAMPLED_EXP{0:d}_fix.fits".format(exp +
1)
imageselfcal = "IMAGE_RESAMPLED_EXP{0:d}_fix.fits".format(exp + 1)
if not os.path.isfile(cubeselfcal):
print('Reconstruct cube {}'.format(cubeselfcal))
#now reconstruct cube and white image on right reference frame
#handle sof file
sof_name = "../../Script/scipost_mpdaf_self{0:d}.sof".format(
exp + 1)
sofedit = open(sof_name, 'w')
sofedit.write(
'../../../{}/DATACUBE_FINAL.fits OUTPUT_WCS\n'.format(
refpath))
sofedit.write(
"PIXTABLE_REDUCED_RESAMPLED_EXP{0:d}_fix.fits PIXTABLE_OBJECT\n"
.format(exp + 1))
sofedit.close()
#now run script
scr = open(
"../../Script/make_scipost_mpdaf_self{0:d}.sh".format(exp +
1),
"w")
scr.write("OMP_NUM_THREADS={0:d}\n".format(nproc))
scr.write(
'esorex --log-file=scipost_mpdaf_self{0:d}.log muse_scipost_make_cube ../../Script/scipost_mpdaf_self{0:d}.sof'
.format(exp + 1))
scr.close()
#Run pipeline
subprocess.call([
"sh",
"../../Script/make_scipost_mpdaf_self{0:d}.sh".format(exp +
1)
])
subprocess.call(["mv", "DATACUBE_FINAL.fits", cubeselfcal])
subprocess.call(["mv", "IMAGE_FOV_0001.fits", imageselfcal])
else:
print('Cube {} already exists! Skip...'.format(cubeselfcal))
#back to top
os.chdir(topdir)
def mask_pixtable(self, mask_name=None, **kwargs):
"""Use the Image Mask and create a new Pixtable
Input
-----
mask_name: str
Name of the mask to be used (FITS file)
use_folder: bool
If True, use the same folder as the Pixtable
Otherwise just write where you stand
suffix_out: str
Suffix for the name of the output Pixtable
If provided, will overwrite the one in self.suffix_out
"""
# Open the PixTable
upipe.print_info("Opening the Pixtable {0}".format(self.pixtable_name))
pixtable = PixTable(self.pixtable_name)
# Use the Image mask and create a pixtable mask
if mask_name is not None:
self.mask_name = mask_name
else:
if not hasattr(self, "mask_name"):
upipe.print_error("Please provide a mask name (FITS file)")
return
upipe.print_info("Creating a column Mask from file {0}".format(
self.mask_name))
mask_col = pixtable.mask_column(self.mask_name)
# extract the right data using the pixtable mask
upipe.print_info("Extracting the Mask")
newpixtable = pixtable.extract_from_mask(mask_col.maskcol)
# Rewrite a new pixtable
self.suffix_out = kwargs.pop("suffix_out", self.suffix_out)
use_folder = kwargs.pop("use_folder", True)
if use_folder:
self.newpixtable_name = joinpath(
self.pixtable_folder, "{0}{1}".format(self.suffix_out,
self.pixtable_name))
else:
self.newpixtable_name = "{0}{1}".format(self.suffix_out,
self.pixtable_name)
upipe.print_info("Writing the new PixTable in {0}".format(
self.newpixtable_name))
newpixtable.write(self.newpixtable_name)
# Now transfer the flat field if it exists
ext_name = 'PIXTABLE_FLAT_FIELD'
try:
# Test if Extension exists by reading header
# If it exists then do nothing
test_data = pyfits.getheader(self.newpixtable_name, ext_name)
upipe.print_warning(
"Flat field extension already exists in masked PixTable - all good"
)
# If it does not exist test if it exists in the original PixTable
except KeyError:
try:
# Read data and header
ff_ext_data = pyfits.getdata(self.pixtable_name, ext_name)
ff_ext_h = pyfits.getheader(self.pixtable_name, ext_name)
upipe.print_warning(
"Flat field extension will be transferred from PixTable")
# Append it to the new pixtable
pyfits.append(self.newpixtable_name, ff_ext_data, ff_ext_h)
except KeyError:
upipe.print_warning(
"No Flat field extension to transfer - all good")
except:
pass
except:
pass
# Patch to fix the extension names of the PixTable
# We have to put a number of extension in lowercase to make sure
# the MUSE recipes understand them
descl = ['xpos', 'ypos', 'lambda', 'data', 'dq', 'stat', 'origin']
for d in descl:
try:
pyfits.setval(self.newpixtable_name,
keyword='EXTNAME',
value=d,
extname=d.upper())
upipe.print_warning(
"Rewriting extension name {0} as lowercase".format(
d.upper()))
except:
upipe.print_warning(
"Extension {0} not present - patch ignored".format(
d.upper()))