def test_select_sky(numexpr, shape, pixfile): pix = PixTable(pixfile) with toggle_numexpr(numexpr): x, y = pix.get_pos_sky() cx = (x.min() + x.max()) / 2 cy = (y.min() + y.max()) / 2 pix2 = pix.extract(sky=(cy, cx, 12, shape)) assert pix2.nrows < pix.nrows
def test_reconstruct(pixfile): pix = PixTable(pixfile).extract(ifu=1) im = pix.reconstruct_det_image() assert im.shape == (1101, 1920) im = pix.reconstruct_det_waveimage() assert im.shape == (1101, 1920)
def test_select(numexpr, pixfile): pix = PixTable(pixfile) with toggle_numexpr(numexpr): pix2 = pix.extract(xpix=[(1000, 2000)], ypix=[(1000, 2000)]) origin = pix2.get_origin() xpix = pix2.origin2xpix(origin) ypix = pix2.origin2ypix(origin) assert xpix.min() >= 1000 assert xpix.max() <= 2000 assert ypix.min() >= 1000 assert ypix.max() <= 2000
def test_write(self): tmpdir = tempfile.mkdtemp(suffix='.mpdaf-test-pixtable') out = join(tmpdir, 'PIX.fits') self.pix.write(out) pix1 = PixTable(out) out = join(tmpdir, 'PIX2.fits') self.pix.write(out, save_as_ima=False) pix2 = PixTable(out) for p in (pix1, pix2): self.assertEqual(self.pix.nrows, p.nrows) for name in ('xpos', 'ypos', 'data', 'dq', 'stat', 'origin'): assert_allclose( getattr(p, 'get_' + name)(), getattr(self, name)) pix1.hdulist.close() pix2.hdulist.close()
def setUp(self): np.random.seed(42) self.xpos = np.linspace(1, 10, NROWS) self.ypos = np.linspace(2, 6, NROWS) self.lbda = np.linspace(5000, 8000, NROWS) self.data = np.linspace(0, 100, NROWS) self.dq = np.random.randint(0, 2, NROWS) self.stat = np.linspace(0, 1, NROWS) # generate origin column np.random.seed(42) self.aifu = np.random.randint(1, 25, NROWS) self.aslice = np.random.randint(1, 49, NROWS) self.ax = np.random.randint(1, 4112, NROWS) self.ay = np.random.randint(1, 4112, NROWS) self.aoffset = self.ax // 90 * 90 # Make sure we have at least one ifu=1 value to avoid random failures self.aifu[0] = 1 self.origin = (((self.ax - self.aoffset) << MUSE_ORIGIN_SHIFT_XSLICE) | (self.ay << MUSE_ORIGIN_SHIFT_YPIX) | (self.aifu << MUSE_ORIGIN_SHIFT_IFU) | self.aslice) prihdu = fits.PrimaryHDU() prihdu.header['author'] = ('MPDAF', 'origin of the file') prihdu.header['RA'] = 0.0 prihdu.header['DEC'] = 0.0 prihdu.header['HIERARCH ESO DRS MUSE PIXTABLE WCS'] = \ 'projected (intermediate)' self.pix = PixTable(None, xpos=self.xpos, ypos=self.ypos, lbda=self.lbda, data=self.data, dq=self.dq, stat=self.stat, origin=self.origin, primary_header=prihdu.header) self.file = io.BytesIO() shape = (NROWS, 1) hdu = fits.HDUList([ prihdu, fits.ImageHDU(name='xpos', data=self.xpos.reshape(shape)), fits.ImageHDU(name='ypos', data=self.ypos.reshape(shape)), fits.ImageHDU(name='lambda', data=self.lbda.reshape(shape)), fits.ImageHDU(name='data', data=self.data.reshape(shape)), fits.ImageHDU(name='dq', data=self.dq.reshape(shape)), fits.ImageHDU(name='stat', data=self.stat.reshape(shape)), fits.ImageHDU(name='origin', data=self.origin.reshape(shape)), ]) hdu[1].header['BUNIT'] = self.pix.wcs.to_string('fits') hdu[2].header['BUNIT'] = self.pix.wcs.to_string('fits') hdu[3].header['BUNIT'] = self.pix.wave.to_string('fits') hdu[4].header['BUNIT'] = self.pix.unit_data.to_string('fits') hdu[6].header['BUNIT'] = (self.pix.unit_data**2).to_string('fits') hdu.writeto(self.file) self.file.seek(0) self.pix2 = PixTable(self.file) self.file.seek(0)
class TestBasicPixTable(unittest.TestCase): @classmethod def setUp(self): np.random.seed(42) self.xpos = np.linspace(1, 10, NROWS) self.ypos = np.linspace(2, 6, NROWS) self.lbda = np.linspace(5000, 8000, NROWS) self.data = np.linspace(0, 100, NROWS) self.dq = np.random.randint(0, 2, NROWS) self.stat = np.linspace(0, 1, NROWS) # generate origin column np.random.seed(42) self.aifu = np.random.randint(1, 25, NROWS) self.aslice = np.random.randint(1, 49, NROWS) self.ax = np.random.randint(1, 4112, NROWS) self.ay = np.random.randint(1, 4112, NROWS) self.aoffset = self.ax // 90 * 90 # Make sure we have at least one ifu=1 value to avoid random failures self.aifu[0] = 1 self.origin = (((self.ax - self.aoffset) << MUSE_ORIGIN_SHIFT_XSLICE) | (self.ay << MUSE_ORIGIN_SHIFT_YPIX) | (self.aifu << MUSE_ORIGIN_SHIFT_IFU) | self.aslice) prihdu = fits.PrimaryHDU() prihdu.header['author'] = ('MPDAF', 'origin of the file') prihdu.header['RA'] = 0.0 prihdu.header['DEC'] = 0.0 prihdu.header['HIERARCH ESO DRS MUSE PIXTABLE WCS'] = \ 'projected (intermediate)' self.pix = PixTable(None, xpos=self.xpos, ypos=self.ypos, lbda=self.lbda, data=self.data, dq=self.dq, stat=self.stat, origin=self.origin, primary_header=prihdu.header) self.file = io.BytesIO() shape = (NROWS, 1) hdu = fits.HDUList([ prihdu, fits.ImageHDU(name='xpos', data=self.xpos.reshape(shape)), fits.ImageHDU(name='ypos', data=self.ypos.reshape(shape)), fits.ImageHDU(name='lambda', data=self.lbda.reshape(shape)), fits.ImageHDU(name='data', data=self.data.reshape(shape)), fits.ImageHDU(name='dq', data=self.dq.reshape(shape)), fits.ImageHDU(name='stat', data=self.stat.reshape(shape)), fits.ImageHDU(name='origin', data=self.origin.reshape(shape)), ]) hdu[1].header['BUNIT'] = self.pix.wcs.to_string('fits') hdu[2].header['BUNIT'] = self.pix.wcs.to_string('fits') hdu[3].header['BUNIT'] = self.pix.wave.to_string('fits') hdu[4].header['BUNIT'] = self.pix.unit_data.to_string('fits') hdu[6].header['BUNIT'] = (self.pix.unit_data**2).to_string('fits') hdu.writeto(self.file) self.file.seek(0) self.pix2 = PixTable(self.file) self.file.seek(0) def test_empty_pixtable(self): pix = PixTable(None) self.assertEqual(pix.nrows, 0) self.assertEqual(pix.extract(), None) self.assertEqual(pix.get_data(), None) def test_header(self): assert self.pix.fluxcal is False assert self.pix.skysub is False assert self.pix.projection == 'projected' pix2 = self.pix.copy() pix2.primary_header['HIERARCH ESO DRS MUSE PIXTABLE FLUXCAL'] = True assert pix2.fluxcal is True def test_getters(self): self.assertEqual(NROWS, self.pix.nrows) for name in ('xpos', 'ypos', 'data', 'dq', 'stat', 'origin'): assert_array_equal( getattr(self.pix, 'get_' + name)(), getattr(self, name)) assert_array_equal( getattr(self.pix2, 'get_' + name)(), getattr(self, name)) def test_get_lambda(self): assert_array_equal(self.lbda, self.pix.get_lambda()) assert_array_equal(self.lbda * u.angstrom.to(u.nm), self.pix.get_lambda(unit=u.nm)) ksel = np.where(self.lbda > 6000) assert_array_equal(self.lbda[ksel], self.pix.get_lambda(ksel)) ksel = (self.lbda > 6000) assert_array_equal(self.lbda[ksel], self.pix.get_lambda(ksel)) assert_array_equal(self.lbda, self.pix2.get_lambda()) ksel = np.where(self.lbda > 6000) assert_array_equal(self.lbda[ksel], self.pix2.get_lambda(ksel)) ksel = (self.lbda > 6000) assert_array_equal(self.lbda[ksel], self.pix2.get_lambda(ksel)) def test_get_xypos(self): assert_array_equal(self.xpos, self.pix2.get_xpos()) assert_array_equal(self.xpos, self.pix.get_xpos(unit=u.pix)) assert_array_equal(self.ypos, self.pix2.get_ypos()) assert_array_equal(self.ypos, self.pix.get_ypos(unit=u.pix)) def test_get_data(self): assert_array_equal(self.data, self.pix2.get_data()) assert_array_equal(self.data, self.pix.get_data(unit=u.count)) assert_array_equal(self.stat, self.pix2.get_stat()) assert_array_equal(self.stat, self.pix.get_stat(unit=u.count**2)) def test_get_row(self): assert self.pix.get_row(0) == { 'xpos': 1, 'ypos': 2, 'lbda': 5000, 'data': 0, 'stat': 0, 'dq': 0, 'origin': 1241548872, } res = self.pix.get_row([0, 1, 2]) assert len(res['xpos']) == 3 assert_array_equal(res['origin'], [1241548872, 908518680, 541389771]) def test_set_column(self): for pixt in (self.pix, self.pix2): pix = pixt.copy() with self.assertRaises(AssertionError): pix.set_xpos(list(range(5))) for pix in (self.pix, self.pix2): new_xpos = np.linspace(2, 3, pix.nrows) pix.set_xpos(new_xpos) assert_array_equal(new_xpos, pix.get_xpos()) new_ypos = np.linspace(2, 3, pix.nrows) pix.set_ypos(new_ypos) assert_array_equal(new_ypos, pix.get_ypos()) new_lambda = np.linspace(4000, 5000, pix.nrows) pix.set_lambda(new_lambda) assert_array_equal(new_lambda, pix.get_lambda()) def test_set_data(self): for pix in (self.pix, self.pix2): new_data = pix.get_data() new_stat = pix.get_stat() ksel = np.where(new_data > 50) new_data[ksel] = 0 new_stat[ksel] = 0 pix.set_data(0, ksel=ksel) assert_array_equal(new_data, pix.get_data()) pix.set_stat(0, ksel=ksel) assert_array_equal(new_stat, pix.get_stat()) new_data = pix.get_data() new_stat = pix.get_stat() ksel = (new_data > 60) new_data[ksel] = 1 new_stat[ksel] = 0 pix.set_data(1, ksel=ksel, unit=u.count) assert_array_equal(new_data, pix.get_data()) pix.set_stat(1, ksel=ksel, unit=u.count**2) assert_array_equal(new_stat, pix.get_stat()) def test_origin_conversion(self): origin = self.pix.get_origin() ifu = self.pix.origin2ifu(origin) sli = self.pix.origin2slice(origin) assert_array_equal(self.aifu, ifu) assert_array_equal(self.aslice, sli) assert_array_equal(self.ay, self.pix.origin2ypix(origin) + 1) # TODO: This one needs a pixtable with a header which contains the # offset values HIERARCH ESO DRS MUSE PIXTABLE EXP0 IFU* SLICE* XOFFSET # assert_array_equal(self.ax, # self.origin2xpix(origin, ifu=ifu, sli=sli)) def test_extract(self): for numexpr in (True, False): with toggle_numexpr(numexpr): pix = self.pix.extract(lbda=(5000, 6000)) ksel = (self.lbda >= 5000) & (self.lbda < 6000) assert_array_equal(self.data[ksel], pix.get_data()) pix = self.pix.extract(ifu=1) ksel = (self.aifu == 1) assert_array_equal(self.data[ksel], pix.get_data()) pix = self.pix.extract(sl=(1, 2, 3)) ksel = ((self.aslice == 1) | (self.aslice == 2) | (self.aslice == 3)) assert_array_equal(self.data[ksel], pix.get_data()) def test_write(self): tmpdir = tempfile.mkdtemp(suffix='.mpdaf-test-pixtable') out = join(tmpdir, 'PIX.fits') self.pix.write(out) pix1 = PixTable(out) out = join(tmpdir, 'PIX2.fits') self.pix.write(out, save_as_ima=False) pix2 = PixTable(out) for p in (pix1, pix2): self.assertEqual(self.pix.nrows, p.nrows) for name in ('xpos', 'ypos', 'data', 'dq', 'stat', 'origin'): assert_allclose( getattr(p, 'get_' + name)(), getattr(self, name)) pix1.hdulist.close() pix2.hdulist.close()
def test_empty_pixtable(self): pix = PixTable(None) self.assertEqual(pix.nrows, 0) self.assertEqual(pix.extract(), None) self.assertEqual(pix.get_data(), None)
def selfcalibrate(listob, deepwhite, refpath='esocombine', nproc=24, extmask=None, extmaskonly=False): """ Loop over each OB and performs self-calibration after masking sources as listob -> OBs to process deepwhite -> the best white image available to mask sources refpath -> where to find a white image to be used as reference wcs grid nproc -> number of processors extmask -> ds9 region file in image coordinates defining the regions to be masked extmaskonly -> if true ose only the mask supplied externally, if false merge sextractor and extmasks """ import os import glob import subprocess import shutil from astropy.io import fits import muse_utils as mut import numpy as np import sep from mpdaf.drs import PixTable from mypython.fits import pyregmask as pmk #grab top dir topdir = os.getcwd() #now loop over each folder and make the final sky-subtracted cubes for ob in listob: #change dir os.chdir(ob + '/Proc/MPDAF') print("Processing {} for self-calibration".format(ob)) #make source mask srcmask = 'selfcalib_mask.fits' if (os.path.isfile(srcmask)): print("Source mask already exists!") else: print("Create source mask for selfcalibration") #open the ifu mask to create a good mask deepdata = fits.open("../../../" + deepwhite) #now flag the sources (allow cubex/eso images) try: image = deepdata[0].data.byteswap().newbyteorder() datheader = deepdata[0].header except: image = deepdata[1].data.byteswap().newbyteorder() datheader = deepdata[1].header bkg = sep.Background(image) bkg.subfrom(image) obj, segmap = sep.extract(image, 5. * bkg.globalrms, minarea=6, segmentation_map=True) segmap[np.where(segmap > 0)] = 1 #write source mask to disk hdu = fits.PrimaryHDU(segmap, header=datheader) hdu.writeto(srcmask, overwrite=True) if (extmask): extfitsmask = 'ext_mask.fits' #Read geometry from automatic mask srchdu = fits.open(srcmask) srchead = srchdu[0].header srcmsk = srchdu[0].data extmsk = np.zeros_like(srcmsk) nx = srchead['NAXIS1'] ny = srchead['NAXIS2'] #construct the sky region mask mysky = pmk.PyMask(nx, ny, "../../../" + extmask, header=srchdu[0].header) for ii in range(mysky.nreg): mysky.fillmask(ii) extmsk = extmsk + mysky.mask outhdu = fits.PrimaryHDU(extmsk, header=srchead) outhdu.writeto(extfitsmask, overwrite=True) srchdu.close() if (extmask and not extmaskonly): namecombmask = 'extsrc_mask.fits' finalmsk = extmsk + srcmsk finalmsk[finalmsk > 0] = 1 outhdu = fits.PrimaryHDU(finalmsk, header=srchead) outhdu.writeto(namecombmask, overwrite=True) finalmask = namecombmask elif (extmask and extmaskonly): finalmask = extfitsmask else: finalmask = srcmask #now loop over exposures and apply self calibration scils = glob.glob("../Basic/OBJECT_RED_0*.fits*") nsci = len(scils) #loop on exposures and apply self calibration for exp in range(nsci): pixselfcal = "PIXTABLE_REDUCED_RESAMPLED_EXP{0:d}_fix.fits".format( exp + 1) if not os.path.isfile(pixselfcal): print("Apply self-calibration to {}".format(pixselfcal)) #open reduced pixel table pix = PixTable( "PIXTABLE_REDUCED_RESAMPLED_EXP{0:d}.fits".format(exp + 1)) #create mask maskpix = pix.mask_column(finalmask) maskpix.write( "PIXTABLE_REDUCED_RESAMPLED_EXP{0:d}_mask.fits".format( exp + 1)) #selfcalibrate autocalib = pix.selfcalibrate(pixmask=maskpix) #write to disk autocalib.write( "PIXTABLE_REDUCED_RESAMPLED_EXP{0:d}_autocalib.fits". format(exp + 1)) pix.write(pixselfcal) else: print("Self-calibration for {} already done! Skip...".format( pixselfcal)) cubeselfcal = "DATACUBE_RESAMPLED_EXP{0:d}_fix.fits".format(exp + 1) imageselfcal = "IMAGE_RESAMPLED_EXP{0:d}_fix.fits".format(exp + 1) if not os.path.isfile(cubeselfcal): print('Reconstruct cube {}'.format(cubeselfcal)) #now reconstruct cube and white image on right reference frame #handle sof file sof_name = "../../Script/scipost_mpdaf_self{0:d}.sof".format( exp + 1) sofedit = open(sof_name, 'w') sofedit.write( '../../../{}/DATACUBE_FINAL.fits OUTPUT_WCS\n'.format( refpath)) sofedit.write( "PIXTABLE_REDUCED_RESAMPLED_EXP{0:d}_fix.fits PIXTABLE_OBJECT\n" .format(exp + 1)) sofedit.close() #now run script scr = open( "../../Script/make_scipost_mpdaf_self{0:d}.sh".format(exp + 1), "w") scr.write("OMP_NUM_THREADS={0:d}\n".format(nproc)) scr.write( 'esorex --log-file=scipost_mpdaf_self{0:d}.log muse_scipost_make_cube ../../Script/scipost_mpdaf_self{0:d}.sof' .format(exp + 1)) scr.close() #Run pipeline subprocess.call([ "sh", "../../Script/make_scipost_mpdaf_self{0:d}.sh".format(exp + 1) ]) subprocess.call(["mv", "DATACUBE_FINAL.fits", cubeselfcal]) subprocess.call(["mv", "IMAGE_FOV_0001.fits", imageselfcal]) else: print('Cube {} already exists! Skip...'.format(cubeselfcal)) #back to top os.chdir(topdir)
def mask_pixtable(self, mask_name=None, **kwargs): """Use the Image Mask and create a new Pixtable Input ----- mask_name: str Name of the mask to be used (FITS file) use_folder: bool If True, use the same folder as the Pixtable Otherwise just write where you stand suffix_out: str Suffix for the name of the output Pixtable If provided, will overwrite the one in self.suffix_out """ # Open the PixTable upipe.print_info("Opening the Pixtable {0}".format(self.pixtable_name)) pixtable = PixTable(self.pixtable_name) # Use the Image mask and create a pixtable mask if mask_name is not None: self.mask_name = mask_name else: if not hasattr(self, "mask_name"): upipe.print_error("Please provide a mask name (FITS file)") return upipe.print_info("Creating a column Mask from file {0}".format( self.mask_name)) mask_col = pixtable.mask_column(self.mask_name) # extract the right data using the pixtable mask upipe.print_info("Extracting the Mask") newpixtable = pixtable.extract_from_mask(mask_col.maskcol) # Rewrite a new pixtable self.suffix_out = kwargs.pop("suffix_out", self.suffix_out) use_folder = kwargs.pop("use_folder", True) if use_folder: self.newpixtable_name = joinpath( self.pixtable_folder, "{0}{1}".format(self.suffix_out, self.pixtable_name)) else: self.newpixtable_name = "{0}{1}".format(self.suffix_out, self.pixtable_name) upipe.print_info("Writing the new PixTable in {0}".format( self.newpixtable_name)) newpixtable.write(self.newpixtable_name) # Now transfer the flat field if it exists ext_name = 'PIXTABLE_FLAT_FIELD' try: # Test if Extension exists by reading header # If it exists then do nothing test_data = pyfits.getheader(self.newpixtable_name, ext_name) upipe.print_warning( "Flat field extension already exists in masked PixTable - all good" ) # If it does not exist test if it exists in the original PixTable except KeyError: try: # Read data and header ff_ext_data = pyfits.getdata(self.pixtable_name, ext_name) ff_ext_h = pyfits.getheader(self.pixtable_name, ext_name) upipe.print_warning( "Flat field extension will be transferred from PixTable") # Append it to the new pixtable pyfits.append(self.newpixtable_name, ff_ext_data, ff_ext_h) except KeyError: upipe.print_warning( "No Flat field extension to transfer - all good") except: pass except: pass # Patch to fix the extension names of the PixTable # We have to put a number of extension in lowercase to make sure # the MUSE recipes understand them descl = ['xpos', 'ypos', 'lambda', 'data', 'dq', 'stat', 'origin'] for d in descl: try: pyfits.setval(self.newpixtable_name, keyword='EXTNAME', value=d, extname=d.upper()) upipe.print_warning( "Rewriting extension name {0} as lowercase".format( d.upper())) except: upipe.print_warning( "Extension {0} not present - patch ignored".format( d.upper()))