def fitsInfo(fitsname=None):
"""
Get fits info
"""
hdu = pyfits.open(fitsname)
hdr = hdu[0].header
ra = hdr['CRVAL1']
dra = abs(hdr['CDELT1'])
raPix = hdr['CRPIX1']
dec = hdr['CRVAL2']
ddec = abs(hdr['CDELT2'])
decPix = hdr['CRPIX2']
freq0 = 0
for i in range(1, hdr['NAXIS']+1):
if hdr['CTYPE%d' % i].strip() == 'FREQ':
freq0 = hdr['CRVAL%d' % i]
break
ndim = hdr["NAXIS"]
imslice = np.zeros(ndim, dtype=int).tolist()
imslice[-2:] = slice(None), slice(None)
image = hdu[0].data[imslice]
wcs = WCS(hdr, mode='pyfits')
return {'image': image, 'wcs': wcs, 'ra': ra, 'dec': dec, 'dra': dra, 'ddec': ddec, 'raPix': raPix, 'decPix': decPix, 'freq0': freq0}
def loadFits(imagename):
with pyfits.open(imagename) as hdu:
data = hdu[0].data
hdr = hdu[0].header
wcs = WCS(hdr, mode="pyfits")
return data, hdr, wcs
def __init__ (self,header):
"""Constructor. Create from filename (treated as FITS file), or a FITS header object""";
# attach to FITS file or header
if isinstance(header,str):
header = pyfits.open(header)[0].header;
else:
self.wcs = WCS(header,mode="pyfits");
try:
ra0,dec0 = self.wcs.getCentreWCSCoords();
self.xpix0,self.ypix0 = self.wcs.wcs2pix(*self.wcs.getCentreWCSCoords());
self.xscale = self.wcs.getXPixelSizeDeg()*DEG;
self.yscale = self.wcs.getYPixelSizeDeg()*DEG;
has_projection = True;
except:
print "No WCS in FITS file, falling back to pixel coordinates.";
ra0 = dec0 = self.xpix0 = self.ypix0 = 0;
self.xscale = self.yscale = DEG/3600;
has_projection = False;
_Projector.__init__(self,ra0*DEG,dec0*DEG,has_projection=has_projection);
def reshape_data (image, prefix=None):
""" Reshape FITS data to (stokes,freq,npix_ra,npix_dec).
Returns reshaped data, wcs, the image header, and
pixel size.
image: Fits data
"""
with pyfits.open(image) as hdu:
data = hdu[0].data
hdr = hdu[0].header
shape = list(data.shape)
ndim = len(shape)
wcs = WCS(hdr, mode="pyfits")
log = logger(level=0, prefix=prefix)
pixel_size = abs(hdr["CDELT1"])
if ndim < 2:
log.error(" The FITS file needs at least two dimensions")
# This is the shape I want the data in
want = (
["STOKES",0],
["FREQ",1],
["RA",2],
["DEC",3],
)
# Assume RA,DEC is first (FITS) or last two (NUMPY)
if ndim > 3:
for ctype, ind in want[:2]:
for axis in range(1, ndim+1):
if hdr["CTYPE%d"%axis].startswith(ctype):
want[ind].append(ndim-axis)
if want[0][-1] == want[1][-2] and want[0][-2] == want[1][-1]:
tmp = shape[0]
shape[0] = shape[1]
shape[1] = tmp
data = numpy.reshape(data,shape)
if ndim == 3:
if not hdr["CTYPE3"].startswith("FREQ"):
data = data[0,...]
elif ndim > 4:
log.error(" FITS file has more than 4 axes. Aborting")
return data, wcs, hdr, pixel_size
def __init__ (self,header):
"""Constructor. Create from filename (treated as FITS file), or a FITS header object""";
# attach to FITS file or header
if isinstance(header,str):
header = pyfits.open(header)[0].header;
else:
self.wcs = WCS(header,mode="pyfits");
try:
ra0,dec0 = self.wcs.getCentreWCSCoords();
self.xpix0,self.ypix0 = self.wcs.wcs2pix(*self.wcs.getCentreWCSCoords());
self.xscale = self.wcs.getXPixelSizeDeg()*DEG;
self.yscale = self.wcs.getYPixelSizeDeg()*DEG;
has_projection = True;
except:
print "No WCS in FITS file, falling back to pixel coordinates.";
ra0 = dec0 = self.xpix0 = self.ypix0 = 0;
self.xscale = self.yscale = DEG/3600;
has_projection = False;
_Projector.__init__(self,ra0*DEG,dec0*DEG,has_projection=has_projection);
class AddBrickDialog(QDialog):
def __init__(self, parent, modal=True, flags=Qt.WindowFlags()):
QDialog.__init__(self, parent, flags)
self.setModal(modal)
self.setWindowTitle("Add FITS brick")
lo = QVBoxLayout(self)
lo.setMargin(10)
lo.setSpacing(5)
# file selector
self.wfile = FileSelector(self,
label="FITS filename:",
dialog_label="FITS file",
default_suffix="fits",
file_types="FITS files (*.fits *.FITS)",
file_mode=QFileDialog.ExistingFile)
lo.addWidget(self.wfile)
# overwrite or add mode
lo1 = QGridLayout()
lo.addLayout(lo1)
lo1.setContentsMargins(0, 0, 0, 0)
lo1.addWidget(QLabel("Padding factor:", self), 0, 0)
self.wpad = QLineEdit("2", self)
self.wpad.setValidator(QDoubleValidator(self))
lo1.addWidget(self.wpad, 0, 1)
lo1.addWidget(QLabel("Assign source name:", self), 1, 0)
self.wname = QLineEdit(self)
lo1.addWidget(self.wname, 1, 1)
# OK/cancel buttons
lo.addSpacing(10)
lo2 = QHBoxLayout()
lo.addLayout(lo2)
lo2.setContentsMargins(0, 0, 0, 0)
lo2.setMargin(5)
self.wokbtn = QPushButton("OK", self)
self.wokbtn.setMinimumWidth(128)
QObject.connect(self.wokbtn, SIGNAL("clicked()"), self.accept)
self.wokbtn.setEnabled(False)
cancelbtn = QPushButton("Cancel", self)
cancelbtn.setMinimumWidth(128)
QObject.connect(cancelbtn, SIGNAL("clicked()"), self.reject)
lo2.addWidget(self.wokbtn)
lo2.addStretch(1)
lo2.addWidget(cancelbtn)
self.setMinimumWidth(384)
# signals
QObject.connect(self.wfile, SIGNAL("filenameSelected"),
self._fileSelected)
# internal state
self.qerrmsg = QErrorMessage(self)
def setModel(self, model):
self.model = model
if model.filename():
self._model_dir = os.path.dirname(os.path.abspath(
model.filename()))
else:
self._model_dir = os.path.abspath('.')
self.wfile.setDirectory(self._model_dir)
self._fileSelected(self.wfile.filename(), quiet=True)
def _fileSelected(self, filename, quiet=False):
self.wokbtn.setEnabled(False)
if not filename:
return None
# check that filename matches model
if not os.path.samefile(self._model_dir, os.path.dirname(filename)):
self.wfile.setFilename('')
if not quiet:
QMessageBox.warning(
self, "Directory mismatch",
"""<P>The FITS file must reside in the same directory
as the current sky model.</P>""")
self.wfile.setDirectory(self._model_dir)
return None
# if filename is not in model already, enable the "add to model" control
for src in self.model.sources:
if isinstance(getattr(src, 'shape', None), ModelClasses.FITSImage):
if os.path.exists(src.shape.filename) and os.path.samefile(
src.shape.filename, filename):
if not quiet:
QMessageBox.warning(
self, "Already in model",
"This FITS brick is already present in the model.")
self.wfile.setFilename('')
return None
if not str(self.wname.text()):
self.wname.setText(
os.path.splitext(os.path.basename(str(filename)))[0])
self.wokbtn.setEnabled(True)
return filename
def accept(self):
"""Tries to add brick, and closes the dialog if successful."""
filename = self.wfile.filename()
# read fits file
busy = BusyIndicator()
try:
input_hdu = pyfits.open(filename)[0]
except Exception, err:
busy = None
QMessageBox.warning(
self, "Error reading FITS",
"Error reading FITS file %s: %s" % (filename, str(err)))
return
# check name
srcname = str(self.wname.text()) or os.path.splitext(
os.path.basename(str(filename)))[0]
if srcname in set([src.name for src in self.model.sources]):
QMessageBox.warning(
self, "Already in model",
"<p>The model already contains a source named '%s'. Please select a different name.</p>"
% srcname)
return
# get image parameters
hdr = input_hdu.header
max_flux = float(input_hdu.data.max())
wcs = WCS(hdr, mode='pyfits')
# Get reference pixel coordinates
# wcs.getCentreWCSCoords() doesn't work, as that gives us the middle of the image
# So scan the header to get the CRPIX values
ra0 = dec0 = 1
for iaxis in range(hdr['NAXIS']):
axs = str(iaxis + 1)
name = hdr.get('CTYPE' + axs, axs).upper()
if name.startswith("RA"):
ra0 = hdr.get('CRPIX' + axs, 1) - 1
elif name.startswith("DEC"):
dec0 = hdr.get('CRPIX' + axs, 1) - 1
# convert pixel to degrees
# print ra0,dec0;
ra0, dec0 = wcs.pix2wcs(ra0, dec0)
ra0 *= DEG
dec0 *= DEG
# print ModelClasses.Position.ra_hms_static(ra0);
# print ModelClasses.Position.dec_sdms_static(dec0);
sx, sy = wcs.getHalfSizeDeg()
sx *= DEG
sy *= DEG
nx, ny = input_hdu.data.shape[-1:-3:-1]
pos = ModelClasses.Position(ra0, dec0)
flux = ModelClasses.Flux(max_flux)
shape = ModelClasses.FITSImage(sx,
sy,
0,
os.path.basename(filename),
nx,
ny,
pad=float(str(self.wpad.text()) or "1"))
img_src = SkyModel.Source(srcname, pos, flux, shape=shape)
self.model.setSources(self.model.sources + [img_src])
self.model.emitUpdate(SkyModel.SkyModel.UpdateAll, origin=self)
busy = None
return QDialog.accept(self)
class FITSWCSpix (_Projector):
"""FITS WCS projection, as determined by a FITS header. lm is in pixels (0-based)."""
def __init__ (self,header):
"""Constructor. Create from filename (treated as FITS file), or a FITS header object""";
# attach to FITS file or header
if isinstance(header,str):
header = pyfits.open(header)[0].header;
else:
self.wcs = WCS(header,mode="pyfits");
try:
ra0,dec0 = self.wcs.getCentreWCSCoords();
self.xpix0,self.ypix0 = self.wcs.wcs2pix(*self.wcs.getCentreWCSCoords());
self.xscale = self.wcs.getXPixelSizeDeg()*DEG;
self.yscale = self.wcs.getYPixelSizeDeg()*DEG;
has_projection = True;
except:
print "No WCS in FITS file, falling back to pixel coordinates.";
ra0 = dec0 = self.xpix0 = self.ypix0 = 0;
self.xscale = self.yscale = DEG/3600;
has_projection = False;
_Projector.__init__(self,ra0*DEG,dec0*DEG,has_projection=has_projection);
def lm (self,ra,dec):
if not self.has_projection():
return numpy.sin(ra)/self.xscale,numpy.sin(dec)/self.yscale;
if numpy.isscalar(ra) and numpy.isscalar(dec):
if ra - self.ra0 > math.pi:
ra -= 2*math.pi;
if ra - self.ra0 < -math.pi:
ra += 2*math.pi;
return self.wcs.wcs2pix(ra/DEG,dec/DEG);
else:
if numpy.isscalar(ra):
ra = numpy.array(ra);
ra[ra - self.ra0 > math.pi] -= 2*math.pi;
ra[ra - self.ra0 < -math.pi] += 2*math.pi;
## when fed in arrays of ra/dec, wcs.wcs2pix will return a nested list of
## [[l1,m1],[l2,m2],,...]. Convert this to an array and extract columns.
lm = numpy.array(self.wcs.wcs2pix(ra/DEG,dec/DEG));
return lm[...,0],lm[...,1];
def radec (self,l,m):
if not self.has_projection():
return numpy.arcsin(l*self.xscale),numpy.arcsin(m*self.yscale);
if numpy.isscalar(l) and numpy.isscalar(m):
ra,dec = self.wcs.pix2wcs(l,m);
else:
## this is slow as molasses because of the way astLib.WCS implements the loop. ~120 seconds for 4M pixels
## when fed in arrays of ra/dec, wcs.wcs2pix will return a nested list of
## [[l1,m1],[l2,m2],,...]. Convert this to an array and extract columns.
# radec = numpy.array(self.wcs.pix2wcs(l,m));
# ra = radec[...,0];
# dec = radec[...,1];
### try a faster implementation -- oh well, only a bit faster, ~95 seconds for the same
### can also replace list comprehension with map(), but that doesn't improve things.
### Note also that the final array constructor takes ~10 secs!
radec = numpy.array([ PyWCSTools.wcs.pix2wcs(self.wcs.WCSStructure,x,y) for x,y in zip(l+1,m+1) ]);
ra = radec[...,0];
dec = radec[...,1];
return ra*DEG,dec*DEG;
def offset (self,dra,ddec):
return self.xpix0 - dra/self.xscale,self.ypix0 + ddec/self.xscale;
def __eq__ (self,other):
"""By default, two projections are the same if their classes match, and their ra0/dec0 match."""
return type(self) is type(other) and (self.ra0,self.dec0,self.xpix0,self.ypix0,self.xscale,self.yscale) == (other.ra0,other.dec0,other.xpix0,other.ypix0,other.xscale,other.yscale);
traceback.print_exc();
busy = None;
QMessageBox.warning(self,"Error writing FITS","Error writing FITS file %s: %s"%(filename,str(err)));
return;
changed = False;
sources = self.model.sources;
# remove sources from model if asked to
if self.wdel.isChecked():
sources = [ src for src in sources if not (src.selected and src.typecode == 'pnt') ];
changed = True;
# add image to model if asked to
if self.wadd.isChecked():
hdr = input_hdu.header;
# get image parameters
max_flux = float(input_hdu.data.max());
wcs = WCS(hdr,mode='pyfits');
# Get reference pixel coordinates
# wcs.getCentreWCSCoords() doesn't work, as that gives us the middle of the image
# So scan the header to get the CRPIX values
ra0 = dec0 = 1;
for iaxis in range(hdr['NAXIS']):
axs = str(iaxis+1);
name = hdr.get('CTYPE'+axs,axs).upper();
if name.startswith("RA"):
ra0 = hdr.get('CRPIX'+axs,1)-1;
elif name.startswith("DEC"):
dec0 = hdr.get('CRPIX'+axs,1)-1;
# convert pixel to degrees
ra0,dec0 = wcs.pix2wcs(ra0,dec0);
ra0 *= DEG;
dec0 *= DEG;
class FITSWCSpix (_Projector):
"""FITS WCS projection, as determined by a FITS header. lm is in pixels (0-based)."""
def __init__ (self,header):
"""Constructor. Create from filename (treated as FITS file), or a FITS header object""";
# attach to FITS file or header
if isinstance(header,str):
header = pyfits.open(header)[0].header;
else:
self.wcs = WCS(header,mode="pyfits");
try:
ra0,dec0 = self.wcs.getCentreWCSCoords();
self.xpix0,self.ypix0 = self.wcs.wcs2pix(*self.wcs.getCentreWCSCoords());
self.xscale = self.wcs.getXPixelSizeDeg()*DEG;
self.yscale = self.wcs.getYPixelSizeDeg()*DEG;
has_projection = True;
except:
print "No WCS in FITS file, falling back to pixel coordinates.";
ra0 = dec0 = self.xpix0 = self.ypix0 = 0;
self.xscale = self.yscale = DEG/3600;
has_projection = False;
_Projector.__init__(self,ra0*DEG,dec0*DEG,has_projection=has_projection);
def lm (self,ra,dec):
if not self.has_projection():
return numpy.sin(ra)/self.xscale,numpy.sin(dec)/self.yscale;
if numpy.isscalar(ra) and numpy.isscalar(dec):
if ra - self.ra0 > math.pi:
ra -= 2*math.pi;
if ra - self.ra0 < -math.pi:
ra += 2*math.pi;
return self.wcs.wcs2pix(ra/DEG,dec/DEG);
else:
if numpy.isscalar(ra):
ra = numpy.array(ra);
ra[ra - self.ra0 > math.pi] -= 2*math.pi;
ra[ra - self.ra0 < -math.pi] += 2*math.pi;
## when fed in arrays of ra/dec, wcs.wcs2pix will return a nested list of
## [[l1,m1],[l2,m2],,...]. Convert this to an array and extract columns.
lm = numpy.array(self.wcs.wcs2pix(ra/DEG,dec/DEG));
return lm[...,0],lm[...,1];
def radec (self,l,m):
if not self.has_projection():
return numpy.arcsin(l*self.xscale),numpy.arcsin(m*self.yscale);
if numpy.isscalar(l) and numpy.isscalar(m):
ra,dec = self.wcs.pix2wcs(l,m);
else:
## this is slow as molasses because of the way astLib.WCS implements the loop. ~120 seconds for 4M pixels
## when fed in arrays of ra/dec, wcs.wcs2pix will return a nested list of
## [[l1,m1],[l2,m2],,...]. Convert this to an array and extract columns.
# radec = numpy.array(self.wcs.pix2wcs(l,m));
# ra = radec[...,0];
# dec = radec[...,1];
### try a faster implementation -- oh well, only a bit faster, ~95 seconds for the same
### can also replace list comprehension with map(), but that doesn't improve things.
### Note also that the final array constructor takes ~10 secs!
radec = numpy.array([ PyWCSTools.wcs.pix2wcs(self.wcs.WCSStructure,x,y) for x,y in zip(l+1,m+1) ]);
ra = radec[...,0];
dec = radec[...,1];
return ra*DEG,dec*DEG;
def offset (self,dra,ddec):
return self.xpix0 - dra/self.xscale,self.ypix0 + ddec/self.xscale;
def __eq__ (self,other):
"""By default, two projections are the same if their classes match, and their ra0/dec0 match."""
return type(self) is type(other) and (self.ra0,self.dec0,self.xpix0,self.ypix0,self.xscale,self.yscale) == (other.ra0,other.dec0,other.xpix0,other.ypix0,other.xscale,other.yscale);
def psf_image_correlation(catalog, psfimage, imagedata, header, wcs=None ,
pixelsize=None, corr_region=5, thresh=0.4, tags=None,
coefftag='high_corr', setatr=True, do_high=False,
prefix=None):
""" Computes correlation of the image and PSF image
catalog : Source model, Tigger format.
psfimage : Instrument's Point spread functions Fits data.
imagedata : Fits data
header : Fits header e.g., img=pyfits.open("test.fits")
header=img[0].header
wcs :This class provides methods
for accessing information from the World Coordinate System
(WCS) contained in the header of a FITS image. Conversions
between pixel and WCS coordinates can also be performed.
If not provided it is directly obtained from the Fits header
provided.
pixelsize: float, Default is None.
If not provided then it is directly obtained form a Fits header.
corr_region : int, optional. A default value of 5.
The size of the region to correlate given in beam sizes.
thresh : float, optional. A default value of 0.4.
Correlation threshold. Sources with correlation > threshold
are sources with high correlation.
tags: str, optional. Default is None.
If specified only sources with 'Tag' will be evaluated.
coefftag: str, optional. A Default string is 'high_corr'.
If provided sources with correlation > thresh will be tagged
using the user specified tag.
setatr : bool, optional. Default is True.
If True all sources will be tagged with 'cf'
giving each detection an extra correlation with PSF parameter.
do_high: bool, optional. Default is False.
If True, sources of high correlation are tagged using 'coefftag',
if False no tagging will be made.
"""
model = Tigger.load(catalog)
image_data = imagedata
beam = header["BMAJ"]
psf_data, wcs_psf, psf_hdr, psf_pix = reshape_data(image=psfimage)
shape = image_data.shape
log = logger(level=0, prefix=prefix)
if pixelsize is None:
pixelsize = abs(header["CDELT1"])
if wcs is None:
wcs = WCS(header, mode="pyfits")
bmaj = int(round(beam/pixelsize))
log.info("Beam major= %d degrees"%bmaj)
if bmaj == 0:
log.debug("Beam major axis was read as 0, setting it to 1")
bmaj = 1.0
if not isinstance(corr_region, int):
if isinstance(corr_region, float):
corr_region = int(round(corr_region))
log.debug("Float is provided and int is required,"
"arounding off to the nearest integer")
if corr_region == 0:
log.error("Rounding off to 0. Provide an integer."
"Aborting")
else:
log.error("corr_region must be an integer. Abort")
step = corr_region * bmaj
sources = []
if tags:
log.debug("Only sources with tag %s will be correlated"
"with the PSF"%tags)
sources = filter(lambda src: src.getTag(tags),model.sources)
else:
for src in model.sources:
sources.append(src)
positions_sky = [map(lambda rad: numpy.rad2deg(rad),
(src.pos.ra, src.pos.dec)) for src in sources]
pos = [wcs.wcs2pix(*pos) for pos in positions_sky]
step = [step,step]
ndim = len(shape)
if ndim == 4:
image_data = image_data[0,0,...]
if ndim == 3:
image_data = image_data[0,...]
pdim = len(psf_data.shape)
if pdim == 4:
psf_data = psf_data[0,0,...]
if pdim == 3:
psf_data = psf_data[0,...]
m = 0
for i, (p, src) in enumerate(zip(pos, sources)):
x,y = p
if x>shape[-2] or y>shape[-1] or numpy.array(p).any()<0:
pos.remove(p)
sources.remove(src)
model.sources.remove(src)
m += 1
if (y+step[1] > shape[-1]) or (y-step[1] < 0):
if p in pos:
pos.remove(p)
model.sources.remove(src)
sources.remove(src)
m += 1
if (x+step[0] > shape[-2]) or (x-step[0] < 0):
if p in pos:
pos.remove(p)
model.sources.remove(src)
sources.remove(src)
m += 1
if m > 0:
log.debug("It will be useful to increase the image size,"
"sources with ra+step or dec+step > image size"
"are removed")
central = psf_hdr["CRPIX2"]
psf_region = psf_data[central-step[0] : central+step[0],
central-step[1] : central+step[1]]
psf_region = psf_region.flatten()
corr = []
n = 0
for src, (ra, dec) in zip(sources, pos):
data_region = image_data[dec-step[0] : dec+step[0],
ra-step[1] : ra+step[1]].flatten()
norm_data = (data_region-data_region.min())/(data_region.max()-
data_region.min())
c_region = numpy.corrcoef((norm_data, psf_region))
cf_region = (numpy.diag((numpy.rot90(c_region))**2)
.sum())**0.5/2**0.5
cf = cf_region
if math.isnan(float(cf)) or cf == 0:
model.sources.remove(src)
sources.remove(src)
n += 1
else:
corr.append(cf)
if setatr:
src.setAttribute("cf",cf)
if n > 0:
log.debug("%d sources were removed due to 0/nan correlation"%n)
thresh = thresh
coefftag = coefftag
if do_high:
for src, crr in zip(sources, corr):
if crr > thresh:
src.setTag(coefftag, True)
model.save(catalog)
return corr
def __init__(self, imagename, psfname, pmodel, nmodel, local_step=10,
snr_thresh=40, high_corr_thresh=0.5, negdetec_region=10,
negatives_thresh=5, phasecenter_excl_radius=None,
prefix=None, loglevel=0):
""" Determines sources that require direction-dependent (DD)
calibration solutions.
psfname: PSF fits data
pmodel: Model of the positive image.
nmodel: Model of the negative image
header: The header of the input image
snr_thresh: float, optional. Default is 40.
Any source with 40 x the minimum SNR.
high_corr_thresh: float, optional. Default is 0.5.
Sources of high PSF correlation have correlation above 0.5.
negdetec_region: float, optional. Default is 10.
Region to lookup for negative detections around. In beam size.
negative_thresh: float, optional. Default is 5.
The number of nearby negative detections. Sources
with number > 5 require direction
dependent (DD) calibration solutions.
phasecenter_excl_region: float (in degrees), optional.
A radius from the phase center (in beam sizes) to exclude the sources
from the evaluation.
prefix: str, optional. Sets a prefix to the output directory.
loglevel: int, optional. Default 0. Python logging.
0, 1, 2, 3 for info, debug, error and
critical, respectively.
"""
self.loglevel = loglevel
self.prefix = prefix
self.log = utils.logger(self.loglevel, prefix=self.prefix)
#image and psf image
self.pmodel = pmodel
self.nmodel = nmodel
self.psfname = psfname
self.log.info(" Loading image data")
# tags
self.dd_tag = "dE"
# thresholds
self.snr_factor = snr_thresh
#self.localthresh = local_thresh
self.corrthresh = high_corr_thresh
self.negthresh = negatives_thresh
with pyfits.open(imagename) as hdu:
self.hdr = hdu[0].header
self.data = utils.image_data(hdu[0].data)
self.wcs = WCS(self.hdr, mode="pyfits")
self.locstep = local_step
#regions
self.phaserad = phasecenter_excl_radius # radius from the phase center
self.negregion = negdetec_region # region to look for negatives
# conversion
self.bmaj = self.hdr["BMAJ"] # in degrees
self.ra0 = numpy.deg2rad(self.hdr["CRVAL1"])
self.dec0 = numpy.deg2rad(self.hdr["CRVAL2"])
def stackem(imagename, positions, width, pixels=False, stokes_ind=0, prefix="stacker-default"):
""" Perfom continuun stacking """
with pyfits.open(imagename) as hdu:
hdr = hdu[0].header
_data = hdu[0].data
ndim = hdr["NAXIS"]
# I only want the image data in the first plane
# May deal with multiple channels/polarizations later
sind = ndim - filter( lambda a: hdr["CTYPE%d"%a].startswith("STOKES"), range(1, ndim+1))[0]
imslice = [slice(None)]*ndim
imslice[:-2] = [0]*(ndim-2)
imslice[sind] = stokes_ind
data = _data[imslice]
if not pixels:
wcs = WCS(hdr, mode="pyfits")
positions = [wcs.wcs2pix(*pos) for pos in positions]
if width%2==1:
width += 1
stacked = numpy.zeros([width,width])
for pos in positions:
stacked += subregion(data, pos, width/2)
# Get average stacked signal
stacked /= len(positions)
# Fit gaussian to stacked emission
# First create a threshhold mask
std = stacked.std()
smooth = filters.gaussian_filter(stacked,[2,2])
mask = smooth>2*std
#p = gaussfitter2.gaussfit(stacked)
#peak = p[1]
#cell = numpy.deg2rad(abs(hdr["CDELT1"]))
#tot = peak*(2*math.pi)*abs(p[4]*p[5])
pylab.imshow(stacked)
pylab.colorbar()
#text = "Peak=%.4g : Tot=%s"%(peak, tot)
#pylab.title(text)
x,y = [numpy.linspace(0,width,width)]*2
xx,yy = numpy.meshgrid(x,y)
#pylab.contour( gaussfitter2.twodgaussian(p,0,1,1)(xx,yy))
pylab.savefig(prefix+"2d.png")
pylab.clf()
pylab.plot(numpy.diagonal(stacked)*1e3, "b-", label="stacked")
noise = data.std()
pylab.plot( [noise*1e3],"r--", label="noise")
pylab.ylabel("mJy/beam")
pylab.legend()
pylab.title("Noise = %.3g mJy"%(noise*1e3))
pylab.savefig(prefix+"1d.png")
pylab.clf()
return stacked.max()
def main():
# setup some standard command-line option parsing
#
from optparse import OptionParser
parser = OptionParser(usage="""%prog: [options] <image names...>""")
parser.add_option("-o",
"--output",
dest="output",
type="string",
help="name of output FITS file")
parser.add_option(
"-r",
"--replace",
action="store_true",
help="replace (first) input file by output. Implies '--force'.")
parser.add_option("-f",
"--force",
dest="force",
action="store_true",
help="overwrite output file even if it exists")
parser.add_option(
"-S",
"--sanitize",
type="float",
metavar="VALUE",
help="sanitize FITS files by replacing NANs and INFs with VALUE")
parser.add_option("-N",
"--nonneg",
action="store_true",
help="replace negative values by 0")
parser.add_option("-m",
"--mean",
dest="mean",
action="store_true",
help="take mean of input images")
parser.add_option("-d",
"--diff",
dest="diff",
action="store_true",
help="take difference of 2 input images")
parser.add_option(
"-t",
"--transfer",
action="store_true",
help=
"transfer data from image 2 into image 1, preserving the FITS header of image 1"
)
parser.add_option("-z",
"--zoom",
dest="zoom",
type="int",
metavar="NPIX",
help="zoom into central region of NPIX x NPIX size")
parser.add_option("-R",
"--rescale",
dest="rescale",
type="float",
help="rescale image values")
parser.add_option(
"-E",
"--edit-header",
metavar="KEY=VALUE",
type="string",
action="append",
help=
"replace header KEY with VALUE. Use KEY=VALUE for floats and KEY='VALUE' for strings."
)
parser.add_option("-D",
"--delete-header",
metavar="KEY",
type="string",
action="append",
help="Remove KEY from header")
parser.add_option(
"--stack",
metavar="outfile:axis",
help=
"Stack a list of FITS images along a given axis. This axis may given as an integer"
"(as it appears in the NAXIS keyword), or as a string (as it appears in the CTYPE keyword)"
)
parser.add_option("--reorder",
help="Required order. List of comma seperated indeces")
parser.add_option(
"--add-axis",
metavar="CTYPE:CRVAL:CRPIX:CDELT[:CUNIT:CROTA]",
action="append",
default=[],
help=
"Add axis to a FITS image. The AXIS will be described by CTYPE:CRVAL:CRPIX:CDELT[:CUNIT:CROTA]. "
"The keywords in brackets are optinal, while those not in brackets are mendatory. "
"This axis will be the last dimension. Maybe specified more than once."
)
parser.add_option(
"--unstack",
metavar="prefix:axis:each_chunk",
help=
"Unstack a FITS image into smaller chunks each having [each_chunk] planes along a given axis. "
"This axis may given as an integer (as it appears in the NAXIS keyword), or as a string "
"(as it appears in the CTYPE keyword)")
parser.add_option("-H",
"--header",
action="store_true",
help="print header(s) of input image(s)")
parser.add_option(
"-s",
"--stats",
action="store_true",
help="print stats on images and exit. No output images will be written."
)
parser.set_defaults(output="",
mean=False,
zoom=0,
rescale=1,
edit_header=[],
delete_header=[])
(options, imagenames) = parser.parse_args()
if not imagenames:
parser.error("No images specified. Use '-h' for help.")
# print "%d input image(s): %s"%(len(imagenames),", ".join(imagenames));
images = [pyfits.open(img) for img in imagenames]
updated = False
# Stack FITS images
if options.stack:
if len(imagenames) < 1:
parser.error("Need more than one image to stack")
stack_args = options.stack.split(":")
if len(stack_args) != 2:
parser.error(
"Two --stack options are required. See ./fitstool.py -h")
outfile, axis = stack_args
_string = True
try:
axis = int(axis)
_string = False
except ValueError:
_string = True
stack_planes(imagenames,
ctype=axis if _string else None,
keep_old=True,
axis=None if _string else axis,
outname=outfile,
fits=True)
# Unstack FITS image
if options.unstack:
image = imagenames[0]
if len(imagenames) < 1:
parser.error("Need more than one image to stack")
unstack_args = options.unstack.split(":")
if len(unstack_args) != 3:
parser.error(
"Two --unstack options are required. See ./fitstool.py -h")
prefix, axis, each_chunk = unstack_args
_string = True
try:
axis = int(axis)
_string = False
except ValueError:
_string = True
each_chunk = int(each_chunk)
unstack_planes(image,
each_chunk,
ctype=axis if _string else None,
axis=None if _string else axis,
prefix=prefix,
fits=True)
sys.exit(0)
if options.add_axis:
for axis in options.add_axis:
hdu = pyfits.open(imagenames[0])
hdr = hdu[0].header
ndim = hdr["NAXIS"]
hdu[0].data = hdu[0].data[numpy.newaxis, ...]
_mendatory = "CTYPE CRVAL CDELT CRPIX".split()
_optional = "CUNIT CROTA".split()
L = len(_mendatory + _optional)
values = axis.split(":")
Lv = len(values)
if len(_mendatory) > len(values):
parser.error(
"Something with the way specified --add-axis. See %prog -h for help"
)
for i, value in enumerate(values):
try:
value = float(value)
except ValueError:
if isinstance(value, str):
value = value.upper()
hdu[0].header["%s%d" %
((_mendatory + _optional)[i], ndim + 1)] = value
hdu.writeto(imagenames[0], clobber=True)
print("Successfully added axis %s to %s" %
(values[0], imagenames[0]))
if options.reorder:
for image in imagenames:
order = map(int, options.reorder.split(","))
reorder(image, order=order, outfile=image)
if options.header:
for filename, img in zip(imagenames, images):
img.verify('silentfix')
if len(imagenames) > 1:
print "======== FITS header for", filename
for hdrline in img[0].header.cards:
print hdrline
if options.replace or len(imagenames) < 2:
if options.output:
parser.error("Cannot combine -r/--replace with -o/--output")
outname = imagenames[0]
options.force = True
autoname = False
else:
outname = options.output
autoname = not outname
if autoname:
outname = re.split('[_]', imagenames[0], 1)[-1]
for keyval in options.edit_header:
key, val = keyval.split("=")
q = ''
if val[0] == "'" and val[-1] == "'":
images[0][0].header[key] = val[1:-1:]
q = '"'
elif val[-1] == 'd' or key.startswith('NAXIS'):
images[0][0].header[key] = int(val[:-1] if val[-1] == 'd' else val)
else:
try:
images[0][0].header[key] = float(val)
except:
images[0][0].header[key] = val
q = '"'
print "Setting header %s=%s%s%s" % (key, q, val, q)
updated = True
for key in options.delete_header:
try:
del images[0][0].header[key]
except KeyError:
raise "Key '%s' not found in FITS header" % key
print "Deleting key '%s' header" % key
updated = True
if options.sanitize is not None:
print "Sanitizing: replacing INF/NAN with", options.sanitize
for img in images:
d = img[0].data
d[numpy.isnan(d) + numpy.isinf(d)] = options.sanitize
# if using stats, do not generate output
if not options.stats:
updated = True
if options.nonneg:
print "Replacing negative value by 0"
for img, name in zip(images, imagenames)[:1]:
d = img[0].data
wh = d < 0
d[wh] = 0
print "Image %s: replaced %d points" % (name, wh.sum())
updated = True
if options.transfer:
if len(images) != 2:
parser.error(
"The --transfer option requires exactly two input images.")
if autoname:
outname = "xfer_" + outname
print "Transferring %s into coordinate system of %s" % (imagenames[1],
imagenames[0])
images[0][0].data = images[1][0].data
updated = True
elif options.diff:
if len(images) != 2:
parser.error(
"The --diff option requires exactly two input images.")
if autoname:
outname = "diff_" + outname
print "Computing difference"
data = images[0][0].data
data -= images[1][0].data
updated = True
elif options.mean:
if autoname:
outname = "mean%d_" % len(images) + outname
print "Computing mean"
data = images[0][0].data
for img in images[1:]:
data += img[0].data
data /= len(images)
images = [images[0]]
updated = True
if options.zoom:
z = options.zoom
if autoname:
outname = "zoom%d_" % z + outname
if len(images) > 1:
"Too many input images specified for this operation, at most 1 expected"
sys.exit(2)
data = images[0][0].data
nx = data.shape[-2]
ny = data.shape[-1]
zdata = data[:, :, (nx - z) / 2:(nx + z) / 2,
(ny - z) / 2:(ny + z) / 2]
#update header
hdr = images[0][0].header
wcs = WCS(hdr, mode="pyfits")
cr1, cr2 = wcs.pix2wcs(ny / 2, nx / 2)
hdr["CRVAL1"] = cr1
hdr["CRVAL2"] = cr2
hdr["CRPIX1"] = zdata.shape[-1] / 2
hdr["CRPIX2"] = zdata.shape[-2] / 2
print "Making zoomed image of shape", "x".join(map(str, zdata.shape))
images = [pyfits.PrimaryHDU(zdata, hdr)]
updated = True
if options.rescale != 1:
if autoname and not updated:
outname = "rescale_" + outname
if len(images) > 1:
"Too many input images specified for this operation, at most 1 expected"
sys.exit(2)
print "Applying scaling factor of %f to image values" % options.rescale
images[0][0].data *= options.rescale
updated = True
if updated:
imagenames[0] = outname
if options.stats:
for ff, filename in zip(images, imagenames):
data = ff[0].data
min, max, dum1, dum2 = scipy.ndimage.measurements.extrema(data)
sum = data.sum()
mean = sum / data.size
std = math.sqrt(((data - mean)**2).mean())
print "%s: min %g, max %g, sum %g, np %d, mean %g, std %g" % (
filename, min, max, sum, data.size, mean, std)
sys.exit(0)
if updated:
print "Writing output image", outname
if os.path.exists(outname) and not options.force:
print "Output image exists, rerun with the -f switch to overwrite."
sys.exit(1)
images[0].writeto(outname, clobber=True)
elif not (options.header or options.stack or options.add_axis
or options.reorder):
print "No operations specified. Use --help for help."
def correlation_factor(src, psf, img, pos_sky, step=None):
image = pf.open(img)
psf_image = pf.open(psf)
hdr = image[0].header
psf_hdr = psf_image[0].header
ndim = len(image[0].shape)
if ndim == 4:
image_data = image[0].data[0, 0, :, :]
psf_data_ = psf_image[0].data[0, 0, :, :]
if ndim == 3:
image_data = image[0].data[0, :, :]
psf_data_ = psf_image[0].data[0, :, :]
if ndim == 2:
image_data = image[0].data[:, :]
psf_data_ = psf_image[0].data[:, :]
elif ndim < 2:
raise ValueError('The FITS file needs at least two dimensions')
elif ndim > 4:
raise ValueError('FITS file has more than 4 axes. Aborting')
# image padding
n_pix = hdr['NAXIS1']
padding = np.zeros([n_pix + 2.0 * step, n_pix + 2.0 * step])
padding[step:-step, step:-step] = image_data
hdr['CRPIX1'], hdr['CRPIX2'] = (n_pix + 2.0 * step) / 2.0, (
n_pix + 2.0 * step) / 2.0
# taking the positions
wcs = WCS(hdr, mode='pyfits')
pos_sky = [pos_sky]
pos = [wcs.wcs2pix(*position) for position in pos_sky]
step = [step, step]
wcs_psf = WCS(psf_hdr, mode='pyfits')
#pf.writeto('expore.fits', padding, header=hdr, clobber=True)
center_psf_ra0, center_psf_dec0 = wcs_psf.wcs2pix(psf_hdr['CRVAL1'],
psf_hdr['CRVAL2'])
psf_region = psf_data_[center_psf_dec0 - step[0]:center_psf_dec0 + step[0],
center_psf_ra0 - step[1]:center_psf_ra0 + step[1]]
psf_data = psf_region.flatten()
for (ra, dec) in pos:
data = padding[dec - step[0]:dec + step[0], ra - step[1]:ra + step[1]]
data_region = data.flatten()
#if len(data_region) and len(psf_data) > 1:
if data_region.shape == psf_data.shape:
# computing th correlation matrix and cf is the correlation factor
cmatrix = np.corrcoef((data_region, psf_data))
if len(cmatrix) > 0:
cf = (np.diag((np.rot90(cmatrix))**2).sum())**0.5 / 2**0.5
else:
cf = 0.0000001
src.setTag('cf', cf)
return cf
def __init__(self,
imagename,
psfname,
pmodel,
nmodel,
local_step=10,
snr_thresh=40,
high_corr_thresh=0.5,
negdetec_region=10,
negatives_thresh=5,
phasecenter_excl_radius=None,
prefix=None,
loglevel=0):
""" Determines sources that require direction-dependent (DD)
calibration solutions.
psfname: PSF fits data
pmodel: Model of the positive image.
nmodel: Model of the negative image
header: The header of the input image
snr_thresh: float, optional. Default is 40.
Any source with 40 x the minimum SNR.
high_corr_thresh: float, optional. Default is 0.5.
Sources of high PSF correlation have correlation above 0.5.
negdetec_region: float, optional. Default is 10.
Region to lookup for negative detections around. In beam size.
negative_thresh: float, optional. Default is 5.
The number of nearby negative detections. Sources
with number > 5 require direction
dependent (DD) calibration solutions.
phasecenter_excl_region: float (in degrees), optional.
A radius from the phase center (in beam sizes) to exclude the sources
from the evaluation.
prefix: str, optional. Sets a prefix to the output directory.
loglevel: int, optional. Default 0. Python logging.
0, 1, 2, 3 for info, debug, error and
critical, respectively.
"""
self.loglevel = loglevel
self.prefix = prefix
self.log = utils.logger(self.loglevel, prefix=self.prefix)
#image and psf image
self.pmodel = pmodel
self.nmodel = nmodel
self.psfname = psfname
self.log.info(" Loading image data")
# tags
self.dd_tag = "dE"
# thresholds
self.snr_factor = snr_thresh
#self.localthresh = local_thresh
self.corrthresh = high_corr_thresh
self.negthresh = negatives_thresh
with pyfits.open(imagename) as hdu:
self.hdr = hdu[0].header
self.data = utils.image_data(hdu[0].data)
self.wcs = WCS(self.hdr, mode="pyfits")
self.locstep = local_step
#regions
self.phaserad = phasecenter_excl_radius # radius from the phase center
self.negregion = negdetec_region # region to look for negatives
# conversion
self.bmaj = self.hdr["BMAJ"] # in degrees
self.ra0 = numpy.deg2rad(self.hdr["CRVAL1"])
self.dec0 = numpy.deg2rad(self.hdr["CRVAL2"])
class load(object):
def __init__(self,
imagename,
psfname,
pmodel,
nmodel,
local_step=10,
snr_thresh=40,
high_corr_thresh=0.5,
negdetec_region=10,
negatives_thresh=5,
phasecenter_excl_radius=None,
prefix=None,
loglevel=0):
""" Determines sources that require direction-dependent (DD)
calibration solutions.
psfname: PSF fits data
pmodel: Model of the positive image.
nmodel: Model of the negative image
header: The header of the input image
snr_thresh: float, optional. Default is 40.
Any source with 40 x the minimum SNR.
high_corr_thresh: float, optional. Default is 0.5.
Sources of high PSF correlation have correlation above 0.5.
negdetec_region: float, optional. Default is 10.
Region to lookup for negative detections around. In beam size.
negative_thresh: float, optional. Default is 5.
The number of nearby negative detections. Sources
with number > 5 require direction
dependent (DD) calibration solutions.
phasecenter_excl_region: float (in degrees), optional.
A radius from the phase center (in beam sizes) to exclude the sources
from the evaluation.
prefix: str, optional. Sets a prefix to the output directory.
loglevel: int, optional. Default 0. Python logging.
0, 1, 2, 3 for info, debug, error and
critical, respectively.
"""
self.loglevel = loglevel
self.prefix = prefix
self.log = utils.logger(self.loglevel, prefix=self.prefix)
#image and psf image
self.pmodel = pmodel
self.nmodel = nmodel
self.psfname = psfname
self.log.info(" Loading image data")
# tags
self.dd_tag = "dE"
# thresholds
self.snr_factor = snr_thresh
#self.localthresh = local_thresh
self.corrthresh = high_corr_thresh
self.negthresh = negatives_thresh
with pyfits.open(imagename) as hdu:
self.hdr = hdu[0].header
self.data = utils.image_data(hdu[0].data)
self.wcs = WCS(self.hdr, mode="pyfits")
self.locstep = local_step
#regions
self.phaserad = phasecenter_excl_radius # radius from the phase center
self.negregion = negdetec_region # region to look for negatives
# conversion
self.bmaj = self.hdr["BMAJ"] # in degrees
self.ra0 = numpy.deg2rad(self.hdr["CRVAL1"])
self.dec0 = numpy.deg2rad(self.hdr["CRVAL2"])
def number_negatives(self, source):
#sources = filter(lambda src: src.getTag(tag), psources)
tolerance = numpy.deg2rad(self.negregion * self.bmaj)
if self.phaserad:
radius = numpy.deg2rad(self.phaserad * self.bmaj)
ra, dec = source.pos.ra, source.pos.dec # in radians
within = self.nmodel.getSourcesNear(ra, dec, tolerance)
if len(within) >= self.negthresh:
if self.phaserad:
if dist(self.ra0, self.dec0, ra, dec)[0] > radius:
source.setTag(self.dd_tag, True)
else:
source.setTag(self.dd_tag, True)
def local_noise(self, pos):
# computing the local noise using MAD
x, y = pos
sub_data = self.data[y - self.locstep:y + self.locstep,
x - self.locstep:x + self.locstep]
noise = numpy.mean(abs(sub_data - numpy.mean(sub_data)))
return noise
def source_selection(self):
sources = self.pmodel.sources
noise, mean = utils.negative_noise(self.data, self.prefix)
for srs in sources:
pos = map(lambda rad: numpy.rad2deg(rad),
(srs.pos.ra, srs.pos.dec))
positions = self.wcs.wcs2pix(*pos)
# local noise, determining SNR using the local noise
# threshold is determined using the global noise
local_noise = self.local_noise(positions)
signal_to_noise = srs.flux.I / local_noise
thresh = self.snr_factor * noise
if signal_to_noise > thresh and srs.rel > 0.99:
if self.psfname:
corr = srs.correlation_factor
if corr > self.corrthresh:
self.number_negatives(srs)
if not self.psfname:
self.number_negatives(srs)
return self.pmodel, self.nmodel
def accept(self):
"""Tries to make a brick, and closes the dialog if successful."""
sources = [src for src in self.model.sources if src.selected and src.typecode == 'pnt']
filename = self.wfile.filename()
if not self._fileSelected(filename):
return
# get PB expression
pbfunc = None
if self.wpb_apply.isChecked():
pbexp = str(self.wpb_exp.text())
try:
pbfunc = eval("lambda r,fq:" + pbexp)
except Exception as err:
QMessageBox.warning(self, "Error parsing PB experssion",
"Error parsing primary beam expression %s: %s" % (pbexp, str(err)))
return
# get frequency
freq = str(self.wfreq.text())
freq = float(freq) * 1e+6 if freq else None
# get pad factor
pad = str(self.wpad.text())
pad = max(float(pad), 1) if pad else 1
# read fits file
busy = BusyIndicator()
try:
input_hdu = pyfits.open(filename)[0]
except Exception as err:
busy = None
QMessageBox.warning(self, "Error reading FITS", "Error reading FITS file %s: %s" % (filename, str(err)))
return
# reset data if asked to
if self.woverwrite.isChecked():
input_hdu.data[...] = 0
# insert sources
Imaging.restoreSources(input_hdu, sources, 0, primary_beam=pbfunc, freq=freq)
# save fits file
try:
# pyfits seems to produce an exception:
# TypeError: formatwarning() takes exactly 4 arguments (5 given)
# when attempting to overwrite a file. As a workaround, remove the file first.
if os.path.exists(filename):
os.remove(filename)
input_hdu.writeto(filename)
except Exception as err:
traceback.print_exc()
busy = None
QMessageBox.warning(self, "Error writing FITS", "Error writing FITS file %s: %s" % (filename, str(err)))
return
changed = False
sources = self.model.sources
# remove sources from model if asked to
if self.wdel.isChecked():
sources = [src for src in sources if not (src.selected and src.typecode == 'pnt')]
changed = True
# add image to model if asked to
if self.wadd.isChecked():
hdr = input_hdu.header
# get image parameters
max_flux = float(input_hdu.data.max())
wcs = WCS(hdr, mode='pyfits')
# Get reference pixel coordinates
# wcs.getCentreWCSCoords() doesn't work, as that gives us the middle of the image
# So scan the header to get the CRPIX values
ra0 = dec0 = 1
for iaxis in range(hdr['NAXIS']):
axs = str(iaxis + 1)
name = hdr.get('CTYPE' + axs, axs).upper()
if name.startswith("RA"):
ra0 = hdr.get('CRPIX' + axs, 1) - 1
elif name.startswith("DEC"):
dec0 = hdr.get('CRPIX' + axs, 1) - 1
# convert pixel to degrees
ra0, dec0 = wcs.pix2wcs(ra0, dec0)
ra0 *= DEG
dec0 *= DEG
sx, sy = wcs.getHalfSizeDeg()
sx *= DEG
sy *= DEG
nx, ny = input_hdu.data.shape[-1:-3:-1]
# check if this image is already contained in the model
for src in sources:
if isinstance(getattr(src, 'shape', None), ModelClasses.FITSImage) and os.path.samefile(
src.shape.filename, filename):
# update source parameters
src.pos.ra, src.pos.dec = ra0, dec0
src.flux.I = max_flux
src.shape.ex, src.shape.ey = sx, sy
src.shape.nx, src.shape.ny = nx, ny
src.shape.pad = pad
break
# not contained, make new source object
else:
pos = ModelClasses.Position(ra0, dec0)
flux = ModelClasses.Flux(max_flux)
shape = ModelClasses.FITSImage(sx, sy, 0, os.path.basename(filename), nx, ny, pad=pad)
img_src = SkyModel.Source(os.path.splitext(os.path.basename(filename))[0], pos, flux, shape=shape)
sources.append(img_src)
changed = True
if changed:
self.model.setSources(sources)
self.model.emitUpdate(SkyModel.SkyModel.UpdateAll, origin=self)
self.parent().showMessage("Wrote %d sources to FITS file %s" % (len(sources), filename))
busy = None
return QDialog.accept(self)
class load(object):
def __init__(self, imagename, psfname, pmodel, nmodel, local_step=10,
snr_thresh=40, high_corr_thresh=0.5, negdetec_region=10,
negatives_thresh=5, phasecenter_excl_radius=None,
prefix=None, loglevel=0):
""" Determines sources that require direction-dependent (DD)
calibration solutions.
psfname: PSF fits data
pmodel: Model of the positive image.
nmodel: Model of the negative image
header: The header of the input image
snr_thresh: float, optional. Default is 40.
Any source with 40 x the minimum SNR.
high_corr_thresh: float, optional. Default is 0.5.
Sources of high PSF correlation have correlation above 0.5.
negdetec_region: float, optional. Default is 10.
Region to lookup for negative detections around. In beam size.
negative_thresh: float, optional. Default is 5.
The number of nearby negative detections. Sources
with number > 5 require direction
dependent (DD) calibration solutions.
phasecenter_excl_region: float (in degrees), optional.
A radius from the phase center (in beam sizes) to exclude the sources
from the evaluation.
prefix: str, optional. Sets a prefix to the output directory.
loglevel: int, optional. Default 0. Python logging.
0, 1, 2, 3 for info, debug, error and
critical, respectively.
"""
self.loglevel = loglevel
self.prefix = prefix
self.log = utils.logger(self.loglevel, prefix=self.prefix)
#image and psf image
self.pmodel = pmodel
self.nmodel = nmodel
self.psfname = psfname
self.log.info(" Loading image data")
# tags
self.dd_tag = "dE"
# thresholds
self.snr_factor = snr_thresh
#self.localthresh = local_thresh
self.corrthresh = high_corr_thresh
self.negthresh = negatives_thresh
with pyfits.open(imagename) as hdu:
self.hdr = hdu[0].header
self.data = utils.image_data(hdu[0].data)
self.wcs = WCS(self.hdr, mode="pyfits")
self.locstep = local_step
#regions
self.phaserad = phasecenter_excl_radius # radius from the phase center
self.negregion = negdetec_region # region to look for negatives
# conversion
self.bmaj = self.hdr["BMAJ"] # in degrees
self.ra0 = numpy.deg2rad(self.hdr["CRVAL1"])
self.dec0 = numpy.deg2rad(self.hdr["CRVAL2"])
def number_negatives(self, source):
#sources = filter(lambda src: src.getTag(tag), psources)
tolerance = numpy.deg2rad(self.negregion * self.bmaj)
if self.phaserad:
radius = numpy.deg2rad(self.phaserad * self.bmaj)
ra, dec = source.pos.ra, source.pos.dec # in radians
within = self.nmodel.getSourcesNear(ra, dec, tolerance)
if len(within) >= self.negthresh:
if self.phaserad:
if dist(self.ra0, self.dec0, ra, dec)[0] > radius:
source.setTag(self.dd_tag, True)
else:
source.setTag(self.dd_tag, True)
def local_noise(self, pos):
# computing the local noise using MAD
x, y = pos
sub_data = self.data[y-self.locstep:y+self.locstep,
x-self.locstep:x+self.locstep]
noise = numpy.mean(abs(sub_data - numpy.mean(sub_data)))
return noise
def source_selection(self):
sources = self.pmodel.sources
noise, mean = utils.negative_noise(self.data, self.prefix)
for srs in sources:
pos = map(lambda rad: numpy.rad2deg(rad),(srs.pos.ra,srs.pos.dec))
positions = self.wcs.wcs2pix(*pos)
# local noise, determining SNR using the local noise
# threshold is determined using the global noise
local_noise = self.local_noise(positions)
signal_to_noise = srs.flux.I/local_noise
thresh = self.snr_factor * noise
if signal_to_noise > thresh and srs.rel > 0.99:
if self.psfname:
corr = srs.correlation_factor
if corr > self.corrthresh:
self.number_negatives(srs)
if not self.psfname:
self.number_negatives(srs)
return self.pmodel, self.nmodel
def accept(self):
"""Tries to add brick, and closes the dialog if successful."""
filename = self.wfile.filename()
# read fits file
busy = BusyIndicator()
try:
input_hdu = pyfits.open(filename)[0]
except Exception as err:
busy.reset_cursor()
QMessageBox.warning(
self, "Error reading FITS",
"Error reading FITS file %s: %s" % (filename, str(err)))
return
# check name
srcname = str(self.wname.text()) or os.path.splitext(
os.path.basename(str(filename)))[0]
if srcname in set([src.name for src in self.model.sources]):
QMessageBox.warning(
self, "Already in model",
"<p>The model already contains a source named '%s'. Please select a different name.</p>"
% srcname)
return
# get image parameters
hdr = input_hdu.header
max_flux = float(input_hdu.data.max())
wcs = WCS(hdr, mode='pyfits')
# Get reference pixel coordinates
# wcs.getCentreWCSCoords() doesn't work, as that gives us the middle of the image
# So scan the header to get the CRPIX values
ra0 = dec0 = 1
for iaxis in range(hdr['NAXIS']):
axs = str(iaxis + 1)
name = hdr.get('CTYPE' + axs, axs).upper()
if name.startswith("RA"):
ra0 = hdr.get('CRPIX' + axs, 1) - 1
elif name.startswith("DEC"):
dec0 = hdr.get('CRPIX' + axs, 1) - 1
# convert pixel to degrees
# print ra0,dec0
ra0, dec0 = wcs.pix2wcs(ra0, dec0)
ra0 *= DEG
dec0 *= DEG
# print ModelClasses.Position.ra_hms_static(ra0)
# print ModelClasses.Position.dec_sdms_static(dec0)
sx, sy = wcs.getHalfSizeDeg()
sx *= DEG
sy *= DEG
nx, ny = input_hdu.data.shape[-1:-3:-1]
pos = ModelClasses.Position(ra0, dec0)
flux = ModelClasses.Flux(max_flux)
shape = ModelClasses.FITSImage(sx,
sy,
0,
os.path.basename(filename),
nx,
ny,
pad=float(str(self.wpad.text()) or "1"))
img_src = SkyModel.Source(srcname, pos, flux, shape=shape)
self.model.setSources(self.model.sources + [img_src])
self.model.emitUpdate(SkyModel.SkyModel.UpdateAll, origin=self)
busy.reset_cursor()
return QDialog.accept(self)
def local_variance(imagedata, header, catalog, wcs=None,
pixelsize=None, tag=None,local_region=5,
noise=None, savefig=True, highvariance_factor=0.8,
high_local_tag=None, neg_side=False,
setatr=True, do_high_loc=False, prefix=None):
""" Calculates the local varience (lv) around a source on
one side of interest.
imagedata : Reshaped Fits data
header : Image header
catalog : Source catalog, in Tigger format.
Source model to compute local variance around.
tag : str, optional.
if specified then the local variance will be
computed for only a subset of sources with a tag,
e.g., 'tag=snr'.
header : Fits header e.g., img=pyfits.open("test.fits")
header=img[0].header
wcs :This class provides methods
for accessing information from the World Coordinate System
(WCS) contained in the header of a FITS image. Conversions
between pixel and WCS coordinates can also be performed.
If not provided it is directly obtained from the Fits header
provided.
pixelsize: float, Default is None.
If not provided then it is directly obtained form a Fits header.
local_region: int, optional. A default value of 5.
Gives a region to compute the local variance in
psf sizes, e.g, 'local_region = 2',
then a region (= 2 * beam size) around a source is used.
highvariance_factor: float, optional. A default value of 0.8.
If highvariance_factor=0.8 is given this means that
the sources with local variance greater than
0.8*image_noise will be tagged 'high_var' if
high_local_tag=None.
high_local_tag : str, optional. A default tag None.
Tag assigned to sources of high local variance.
If None is provided the default tag will be used
i.e 'high_var'. Else it uses the specified tag.
setatr : bool, optional. Default is True.
If True all sources will be tagged with 'l'
giving each detection an extra local variance
parameter.
do_high_loc : bool, optional. Default is False.
If True sources with high local variance will be tagged
using 'localvariance_tag' (see above).
prefix : str, optional. Default is None.
"""
data = imagedata
beam = header["BMAJ"]
if pixelsize is None:
pixelsize = abs(header["CDELT1"])
if wcs is None:
wcs = WCS(header, mode="pyfits")
bmaj = int(round(beam/pixelsize)) # beam size in pixels
log = logger(level=0, prefix=prefix)
if not isinstance(local_region, int):
if isinstance(local_region, float):
local_region = int(round(local_region))
log.debug("Float is provided and int is required,"
"arounding offto the nearest integer")
if local_region == 0:
log.error("It rounded off to zero now,"
"change local_region into an integer."
"Aborting")
else:
log.error("local_region must be an integer. Abort")
step = local_region * bmaj
noise = noise or negative_noise(data)
model = Tigger.load(catalog)
sources = []
if tag:
log.debug("Local variance is only computed"
"for sources with a tag %s are"%
tag)
sources = filter(lambda src: src.getTag(tag), model.sources)
else:
for src in model.sources:
sources.append(src)
positions_sky = [map(lambda rad: numpy.rad2deg(rad),
(src.pos.ra,src.pos.dec)) for src in sources]
positions = [wcs.wcs2pix(*pos) for pos in positions_sky]
shape = data.shape
ndim = len( data.shape)
if ndim == 4:
data = data[0,0,...]
if ndim == 3:
data = data[0,...]
step = [step, step]
m = 0
for i, (pos, src) in enumerate(zip(positions, sources)):
x,y = pos
if x>shape[-2] or y>shape[-1] or numpy.array(pos).any()<0:
positions.remove(pos)
model.sources.remove(src)
sources.remove(src)
m += 1
if (y+step[1] > shape[-1]) or (y-step[1] < 0):
if pos in positions:
positions.remove(pos)
model.sources.remove(src)
sources.remove(src)
m += 1
if (x+step[0] > shape[-2]) or (x-step[0] < 0):
if pos in positions:
positions.remove(pos)
model.sources.remove(src)
sources.remove(src)
m += 1
if m > 0:
log.debug("It will be useful to increase the image size,"
"sources with ra+step or dec+step > image size"
"are removed")
_std = []
if neg_side:
data = -data
log.debug("Using the negative side of the provided image.")
n = 0
for (x, y), srs in zip(positions, sources):
pos = [x,y]
subrgn = data[y-step[0] : y+step[0], x-step[1] : x+step[1]]
subrgn = subrgn[subrgn > 0]
std = subrgn.std()
if math.isnan(float(std)) or _std == 0:
sources.remove(srs)
model.sources.remove(srs)
positions.remove(srs)
n += 1
else:
_std.append(std)
if setatr:
srs.setAttribute("l", std)
if n > 0:
log.debug("Nan encountered %d times. Increase the size of the"
"region or check the image. Otherwise sources with"
"0 or nan are flagged." %n)
def high_variance_sources(
pos, local_variance, noise, model, threshold,
savefig=savefig, prefix=None, localtag=None):
if savefig:
save_fig = prefix + "_variance.png" or catalog.replace(
".lsm.html", ".png")
local = [l/1.0e-6 for l in local_variance]
x = numpy.arange(len(pos))
pylab.figure()
pylab.plot(x, local)
pylab.plot([noise/1.0e-6] * len(local_variance))
localtag = localtag
for i, (pos, src) in enumerate(zip( pos, model.sources)):
if _std[i] > threshold:
src.setTag(localtag, True)
pylab.plot(x[i], local[i], "rD")
pylab.annotate(src.name, xy=(x[i],local[i]))
if savefig:
pylab.ylabel("local variance[$\mu$]")
pylab.savefig(save_fig)
if high_local_tag is None:
high_local_tag = "high_var"
if do_high_loc:
threshold = highvariance_factor * noise
high_variance_sources(positions, _std, noise, model,
threshold=threshold, savefig=savefig,
prefix=prefix, localtag=high_local_tag)
model.save(catalog)
return _std
class load(object):
def __init__(self, imagename, psfname=None, sourcefinder_name='pybdsm',
saveformat="gaul", makeplots=True, do_psf_corr=True,
do_local_var=True, psf_corr_region=5, local_var_region=10,
rel_excl_src=None, pos_smooth=2, neg_smooth=2, loglevel=0,
thresh_pix=5, thresh_isl=3, neg_thresh_isl=3,
neg_thresh_pix=5, reset_rel=None, prefix=None,
do_nearsources=False, savefits=False,
increase_beam_cluster=False, savemask_pos=False,
savemask_neg=False, no_smooth=True, **kw):
""" Takes in image and extracts sources and makes
reliability estimations..
imagename: Fits image
psfname: PSF fits image, optional.
sourcefinder_name: str, optional. Default 'pybdsm'.
Uses source finder specified.
makeplots: bool, optional. Default is True.
Make reliability plots.
do_psf_corr : bool, optional. Default True.
If True, PSF correlation will be added
as an extra parameter for density estimations.
NB: the PSF fits image must be provided.
do_local_var : bool, optional. Default is True.
If True, adds local variance as an extra parameter,
for density estimations.
do_nearsources: boolean. Default is False.
If true it adds number of nearest neighnours as an extra
parameter. It looks for sources around 5 beam sizes.
psf_corr_region : int, optional. Default value is 5.
Data size to correlate around a source, in beam sizes.
local_var_region: int, optional. Default 10.
Data size to compute the local variance in beam sizes.
rel_excl_src : floats, optional. Default is None.
Excludes sources in a specified region
e.g ra, dec, radius in degrees. For
2 regions: ra1, dec1, radius1: ra2, dec2, radius2, etc.
pos_smooth : float, optional. Default 2.
Masking threshold for the positive image.
For default value 2, data peaks < 2 * image noise
are masked.
neg_smooth : float, optional. Default 2.
Similar to pos_smooth but applied to the negative image.
thresh_isl : float, optional. Default is 3.
Threshold for forming islands in the positive image
thresh_pix : float, optional. Default is 5.
Threshold for model fitting, in positive image.
neg_thresh_isl : float, optional. Default is 3.
Simialr to thresh_isl but for negative image.
neg_thresh_pix : float, optional. Default is 5.
Similar to thresh_pix but for negative image.
savefits: boolean. Default is False.
If True a negative image is saved.
reset_rel: boolean. Default is False. If true then
sources with correlation < 0.002 and rel >0.60
have their reliabilities set to 0.
increase_beam_cluster: boolean, optional. If True, sources
groupings will be increase by 20% the beam size. If False,
the actual beam size will be used. Default is False.
savemask_pos: boolean, optional. If true the mask applied on
the positive side of an image after smoothing is saved.
savemask_neg: Similar to savemask_pos but for the negative
side of an image.
loglevel : int, optional. Default is 0.
Provides Pythonlogging options, 0, 1, 2 and 3 are for info, debug,
error and critial respectively.
kw : kward for source extractions. Should be a mapping e.g
kw['thresh_isl'] = 2.0 or kw['do_polarization'] = True
"""
#
self.smoothing = not no_smooth
self.prefix = prefix
# log level
self.loglevel = loglevel
self.log = utils.logger(self.loglevel, prefix=self.prefix)
# image, psf image
self.imagename = imagename
self.psfname = psfname
with pyfits.open(imagename) as hdu:
self.header = hdu[0].header
self.wcs = WCS(self.header, mode="pyfits")
self.pixelsize = abs(self.header["cdelt1"])
self.bmaj = numpy.deg2rad(self.header["BMAJ"])
# boolean optionals
self.makeplots = makeplots
self.do_local_var = do_local_var
self.nearsources = do_nearsources
self.do_psf_corr = do_psf_corr
self.savemaskpos = savemask_pos
self.savemaskneg = savemask_neg
self.savefits = savefits
self.derel = reset_rel
self.log.info("Catalogues will be saved as %s, where srl is source "
" and gaul is Gaussians. "%saveformat)
self.catalogue_format = "." + saveformat
if not self.psfname:
self.log.info(" No psf provided, do_psf_corr is set to False.")
self.do_psf_corr = False
# computing negative noise
self.noise, self.mean = utils.negative_noise(self.imagename, self.prefix)
self.log.info(" The negative noise is %e Jy/beam"%self.noise)
if self.noise == 0:
self.log.debug(" The negative noise is 0, check image")
# source finder initialization
self.sourcefinder_name = sourcefinder_name
self.log.info(" Using %s source finder to extract the sources."%
self.sourcefinder_name)
self.negimage = self.prefix + "_negative.fits"
utils.invert_image(self.imagename, self.negimage)
# smoothing factors
self.pos_smooth = pos_smooth
self.neg_smooth = neg_smooth
# region to evaluate
self.corrstep = psf_corr_region
self.localstep = local_var_region
self.radiusrm = rel_excl_src
self.do_beam = increase_beam_cluster
beam_pix = int(round(numpy.rad2deg(self.bmaj)/self.pixelsize))
self.locstep = self.localstep * beam_pix
self.cfstep = self.corrstep * beam_pix
self.bmin, self.bpa = self.header["BMIN"], self.header["BPA"]
self.opts_pos = {}
if self.do_beam:
bmaj = self.header["BMAJ"]
self.opts_pos["beam"] = (1.2*bmaj, 1.2*self.bmin, self.bpa)
# Pybdsm or source finder fitting thresholds
self.thresh_isl = thresh_isl
self.thresh_pix = thresh_pix
self.opts_pos = dict(thresh_pix=self.thresh_pix,
thresh_isl=self.thresh_isl)
self.opts_pos.update(kw)
self.opts_neg = {}
self.opts_neg.update(kw)
self.neg_thresh_isl = neg_thresh_isl
self.neg_thresh_pix = neg_thresh_pix
self.opts_neg["thresh_isl"] = self.neg_thresh_isl
self.opts_neg["thresh_pix"] = self.neg_thresh_pix
def source_finder(self, image=None, thresh=None, prefix=None,
noise=None, output=None, savemask=None, **kw):
#kw.update(kwards)
tpos = None
naxis = self.header["NAXIS1"]
boundary = numpy.array([self.locstep, self.cfstep])
#trim_box = (boundary.max(), naxis - boundary.max(),
# boundary.max(), naxis - boundary.max())
trim_box = None
# data smoothing
if self.smoothing:
ext = utils.fits_ext(image)
tpos = tempfile.NamedTemporaryFile(suffix="."+ext, dir=".")
tpos.flush()
mask, noise = utils.thresh_mask(image, tpos.name,
thresh=thresh, noise=self.noise,
sigma=True, smooth=True, prefix=prefix,
savemask=savemask)
# using the masked image for forming islands
kw["detection_image"] = tpos.name
kw["blank_limit"] = self.noise/1.0e5
# source extraction
utils.sources_extraction(
image=image, output=output,
sourcefinder_name=self.sourcefinder_name,
trim_box=trim_box,
prefix=self.prefix, **kw)
if tpos:
tpos.close()
def remove_sources_within(self, model):
sources = model.sources
rel_remove = self.radiusrm[0].split(":")
for i in range(len(rel_remove)):
ra, dec, tolerance = rel_remove[i].split(",")
ra_r = numpy.deg2rad(float(ra))
dec_r = numpy.deg2rad(float(dec))
tolerance_r = numpy.deg2rad(float(tolerance))
within = model.getSourcesNear(ra_r, dec_r, tolerance_r)
for src in sorted(sources):
if src in within:
model.sources.remove(src)
return model
def params(self, modelfits):
# reads in source finder output
with pyfits.open(modelfits) as hdu:
data = hdu[1].data
tfile = tempfile.NamedTemporaryFile(suffix=".txt")
tfile.flush()
# writes a catalogue in a temporaty txt file
with open(tfile.name, "w") as std:
std.write("#format:name ra_rad dec_rad i emaj_r emin_r pa_r\n")
model = Tigger.load(tfile.name) # open a tmp. file
peak, total, area, loc, corr = [], [], [], [], []
for i in range(len(data)):
flux = data["Total_flux"][i]
dc_emaj, dc_emin = data["DC_Maj"][i], data["DC_Min"][i]
ra, dec = data["RA"][i], data["DEC"][i]
pa = data["DC_PA"][i]
name = "SRC%d"%i
peak_flux = data["Peak_flux"][i]
posrd = ModelClasses.Position(numpy.deg2rad(ra), numpy.deg2rad(dec))
flux_I = ModelClasses.Polarization(flux, 0, 0, 0)
if dc_emaj == 0 and dc_emin == 0:
shape = None
else:
shape = ModelClasses.Gaussian(numpy.deg2rad(dc_emaj), numpy.deg2rad(dc_emin),
numpy.deg2rad(pa))
srs = SkyModel.Source(name, posrd, flux_I, shape=shape)
# using convolved maj and min for reliability estimate
emaj, emin = data["Maj"][i], data["Min"][i]
# area: find ex and ey if are 0 assign beam size
if emaj or emin == 0:
srcarea = math.pi * (numpy.rad2deg(self.bmaj)) * pow(3600.0, 2) *\
(numpy.rad2deg(self.bmin))
if emaj and emin > 0:
srcarea = emaj * emin * math.pi * pow(3600.0, 2) # arcsecond
# only accepts sources with flux > 0 and not nan RA and DEC
# and local variance
pos = [self.wcs.wcs2pix(*(ra, dec))][0] #positions from deg to pixel
with pyfits.open(self.negimage) as hdu:
negdata = utils.image_data( hdu[0].data )
if flux > 0 and peak_flux > 0 and not math.isnan(float(ra))\
and not math.isnan(float(dec)):
local = utils.compute_local_variance(negdata,
pos, self.locstep)
srs.setAttribute("local_variance", local)
if not math.isnan(float(local)) or local > 0:
if self.psfname:
pdata, psf = utils.compute_psf_correlation(self.imagename,
self.psfname, pos, self.cfstep)
if len(pdata) == len(psf):
c_region = numpy.corrcoef((pdata, psf))
cf = (numpy.diag((numpy.rot90(c_region))**2)
.sum())**0.5/2**0.5
srs.setAttribute("correlation_factor", cf)
corr.append(cf)
model.sources.append(srs)
peak.append(peak_flux)
total.append(flux)
area.append(srcarea)
loc.append(local)
else:
model.sources.append(srs)
peak.append(peak_flux)
total.append(flux)
area.append(srcarea)
loc.append(local)
labels = dict(size=(0, "Log$_{10}$(Source area)"),
peak=(1, "Log$_{10}$( Peak flux [Jy] )"),
tot=(2, "Log$_{10}$( Total flux [Jy] )"))
if self.do_psf_corr:
labels.update( {"coeff":(len(labels),
"Log$_{10}$ (CF)")})
if self.do_local_var:
labels.update( {"local": (len(labels),
"Log$_{10}$(Local Variance)")})
if self.nearsources:
labels.update( {"near": (len(labels),
"Log$_{10}$(Near Sources)")})
nsrc = len(model.sources)
out = numpy.zeros([nsrc, len(labels)])
# returning parameters
for i, src in enumerate(model.sources):
ra, dec = src.pos.ra, src.pos.dec
near = model.getSourcesNear(ra, dec, 5 * self.bmaj)
nonear = len(near)
if self.nearsources:
src.setAttribute("neibours", nonear)
if self.do_psf_corr and self.do_local_var and self.nearsources:
out[i,...] = area[i], peak[i], total[i], corr[i], loc[i], nonear
elif self.do_psf_corr and self.do_local_var and not self.nearsources:
out[i,...] = area[i], peak[i], total[i] , corr[i], loc[i]
elif self.do_psf_corr and self.nearsources and not self.do_local_var:
out[i,...] = area[i], peak[i], total[i] , corr[i], nonear
elif not self.do_psf_corr and self.do_local_var and self.nearsources:
out[i,...] = area[i], peak[i], total[i] , loc[i], nonear
elif self.do_psf_corr and not self.do_local_var and not self.nearsources:
out[i,...] = area[i], peak[i], total[i] , corr[i]
elif not self.do_psf_corr and self.do_local_var and not self.nearsources:
out[i,...] = area[i], peak[i], total[i] , loc[i]
elif not self.do_psf_corr and not self.do_local_var and self.nearsources:
out[i,...] = area[i], peak[i], total[i] , nonear
else:
out[i,...] = area[i], peak[i], total[i]
# removes the rows with 0s
removezeros = (out == 0).sum(1)
output = out[removezeros <= 0, :]
return model, numpy.log10(output), labels
def get_reliability(self):
# finding sources
self.log.info(" Extracting the sources on both sides ")
pfile = self.prefix + self.catalogue_format + ".fits"
nfile = self.prefix + "_negative" + self.catalogue_format + ".fits"
# i need to catch mmap.mmap error here
# running a source finder
self.source_finder(image=self.negimage,
output=nfile, thresh=self.neg_smooth,
savemask=self.savemaskneg,
prefix=self.prefix, **self.opts_neg)
self.source_finder(image=self.imagename,
output=pfile, thresh=self.pos_smooth,
savemask=self.savemaskpos,
prefix=self.prefix, **self.opts_pos)
self.log.info(" Source Finder completed successfully ")
pmodel, positive, labels = self.params(pfile)
nmodel, negative, labels = self.params(nfile)
# setting up a kernel, Gaussian kernel
bandwidth = []
for plane in negative.T:
bandwidth.append(plane.std())
nplanes = len(labels)
cov = numpy.zeros([nplanes, nplanes])
nnsrc = len(negative)
npsrc = len(positive)
self.log.info(" There are %d positive and %d negtive detections "%(npsrc, nnsrc))
if nnsrc == 0 or npsrc ==0:
self.log.error("The resulting array has length of 0 thus cannot compute"
" the reliability. Aborting.")
self.log.info(" Computing the reliabilities ")
for i in range(nplanes):
for j in range(nplanes):
if i == j:
cov[i, j] = bandwidth[i]*((4.0/((nplanes+2)*
nnsrc))**(1.0/(nplanes+4.0)))
self.log.info("The resulting covariance matrix is %r"%cov)
pcov = utils.gaussian_kde_set_covariance(positive.T, cov)
ncov = utils.gaussian_kde_set_covariance(negative.T, cov)
# get number densities
nps = pcov(positive.T) * npsrc
nns = ncov(positive.T) * nnsrc
# define reliability of positive catalog
rel = (nps-nns)/nps
for src, rf in zip(pmodel.sources, rel):
src.setAttribute("rel", rf)
self.log.info(" Saved the reliabilities values.")
# remove sources with poor correlation and high reliability,
# the values are currently arbitrary
if self.do_psf_corr and self.derel:
for s in pmodel.sources:
cf, r = s.correlation_factor, s.rel
if cf < 0.006 and r > 0.60:
s.rel = 0.0
if self.makeplots:
savefig = self.prefix + "_planes.png"
utils.plot(positive, negative, rel=rel, labels=labels,
savefig=savefig, prefix=self.prefix)
# removes sources in a given radius from the phase center
if self.radiusrm:
self.log.info(" Remove sources ra, dec, radius of %r"
" from the phase center" %self.radiusrm)
pmodel = self.remove_sources_within(pmodel)
if not self.savefits:
self.log.info(" Deleting the negative image.")
os.system("rm -r %s"%self.negimage)
# Set field Center
pmodel.ra0, pmodel.dec0 = map(numpy.deg2rad, self.wcs.getCentreWCSCoords())
return pmodel, nmodel, self.locstep
def main():
# setup some standard command-line option parsing
#
from optparse import OptionParser
parser = OptionParser(usage="""%prog: [options] <image names...>""");
parser.add_option("-o","--output",dest="output",type="string",
help="name of output FITS file");
parser.add_option("-r","--replace",action="store_true",
help="replace (first) input file by output. Implies '--force'.");
parser.add_option("-f","--force",dest="force",action="store_true",
help="overwrite output file even if it exists");
parser.add_option("-S","--sanitize",type="float",metavar="VALUE",
help="sanitize FITS files by replacing NANs and INFs with VALUE");
parser.add_option("-N","--nonneg",action="store_true",
help="replace negative values by 0");
parser.add_option("-m","--mean",dest="mean",action="store_true",
help="take mean of input images");
parser.add_option("-d","--diff",dest="diff",action="store_true",
help="take difference of 2 input images");
parser.add_option("-t","--transfer",action="store_true",
help="transfer data from image 2 into image 1, preserving the FITS header of image 1");
parser.add_option("-z","--zoom",dest="zoom",type="int",metavar="NPIX",
help="zoom into central region of NPIX x NPIX size");
parser.add_option("-R","--rescale",dest="rescale",type="float",
help="rescale image values");
parser.add_option("-E","--edit-header",metavar="KEY=VALUE",type="string",action="append",
help="replace header KEY with VALUE. Use KEY=VALUE for floats and KEY='VALUE' for strings.");
parser.add_option("--stack",metavar="outfile:axis",
help="Stack a list of FITS images along a given axis. This axis may given as an integer"
"(as it appears in the NAXIS keyword), or as a string (as it appears in the CTYPE keyword)")
parser.add_option("--reorder",
help="Required order. List of comma seperated indeces")
parser.add_option("--add-axis",metavar="CTYPE:CRVAL:CRPIX:CDELT[:CUNIT:CROTA]",action="append",default=[],
help="Add axis to a FITS image. The AXIS will be described by CTYPE:CRVAL:CRPIX:CDELT[:CUNIT:CROTA]. "
"The keywords in brackets are optinal, while those not in brackets are mendatory. "
"This axis will be the last dimension. Maybe specified more than once.")
parser.add_option("--unstack",metavar="prefix:axis:each_chunk",
help="Unstack a FITS image into smaller chunks each having [each_chunk] planes along a given axis. "
"This axis may given as an integer (as it appears in the NAXIS keyword), or as a string "
"(as it appears in the CTYPE keyword)")
parser.add_option("-H","--header",action="store_true",help="print header(s) of input image(s)");
parser.add_option("-s","--stats",action="store_true",help="print stats on images and exit. No output images will be written.");
parser.set_defaults(output="",mean=False,zoom=0,rescale=1,edit_header=[]);
(options,imagenames) = parser.parse_args();
if not imagenames:
parser.error("No images specified. Use '-h' for help.");
# print "%d input image(s): %s"%(len(imagenames),", ".join(imagenames));
images = [ pyfits.open(img) for img in imagenames ];
updated = False;
# Stack FITS images
if options.stack:
if len(imagenames)<1:
parser.error("Need more than one image to stack")
stack_args = options.stack.split(":")
if len(stack_args) != 2:
parser.error("Two --stack options are required. See ./fitstool.py -h")
outfile,axis = stack_args
_string = True
try:
axis = int(axis)
_string = False
except ValueError:
_string = True
stack_planes(imagenames,ctype=axis if _string else None,keep_old=True,
axis=None if _string else axis,outname=outfile,fits=True)
# Unstack FITS image
if options.unstack:
image = imagenames[0]
if len(imagenames)<1:
parser.error("Need more than one image to stack")
unstack_args = options.unstack.split(":")
if len(unstack_args) != 3:
parser.error("Two --unstack options are required. See ./fitstool.py -h")
prefix,axis,each_chunk = unstack_args
_string = True
try:
axis = int(axis)
_string = False
except ValueError:
_string = True
each_chunk = int(each_chunk)
unstack_planes(image,each_chunk,ctype=axis if _string else None,
axis=None if _string else axis,prefix=prefix,fits=True)
sys.exit(0)
if options.add_axis:
for axis in options.add_axis:
hdu = pyfits.open(imagenames[0])
hdr = hdu[0].header
ndim = hdr["NAXIS"]
hdu[0].data = hdu[0].data[numpy.newaxis,...]
_mendatory = "CTYPE CRVAL CDELT CRPIX".split()
_optional = "CUNIT CROTA".split()
L = len(_mendatory+_optional)
values = axis.split(":")
Lv = len(values)
if len(_mendatory)>len(values) :
parser.error("Something with the way specified --add-axis. See %prog -h for help")
for i,value in enumerate(values ) :
try:
value = float(value)
except ValueError:
if isinstance(value, str):
value = value.upper()
hdu[0].header["%s%d"%( (_mendatory+_optional)[i],ndim+1)] = value
hdu.writeto(imagenames[0],clobber=True)
print("Successfully added axis %s to %s"%(values[0],imagenames[0]))
if options.reorder:
for image in imagenames:
order = map(int, options.reorder.split(","))
reorder(image, order=order, outfile=image)
if options.header:
for filename,img in zip(imagenames,images):
if len(imagenames)>1:
print "======== FITS header for",filename;
for hdrline in img[0].header.ascard:
print hdrline;
if options.replace or len(imagenames)<2:
if options.output:
parser.error("Cannot combine -r/--replace with -o/--output");
outname = imagenames[0];
options.force = True;
autoname = False;
else:
outname = options.output;
autoname = not outname;
if autoname:
outname = re.split('[_]',imagenames[0],1)[-1];
for keyval in options.edit_header:
key,val = keyval.split("=");
q = ''
if val[0] == "'" and val[-1] == "'":
images[0][0].header[key] = val[1:-1:];
q = '"'
elif val[-1] == 'd' or key.startswith('NAXIS'):
images[0][0].header[key] = int(val[:-1] if val[-1]=='d' else val);
else:
try:
images[0][0].header[key] = float(val);
except:
images[0][0].header[key] = val;
q = '"';
print "Setting header %s=%s%s%s"%(key,q,val,q);
updated = True;
if options.sanitize is not None:
print "Sanitizing: replacing INF/NAN with",options.sanitize;
for img in images:
d = img[0].data;
d[numpy.isnan(d)+numpy.isinf(d)] = options.sanitize;
# if using stats, do not generate output
if not options.stats:
updated = True;
if options.nonneg:
print "Replacing negative value by 0";
for img,name in zip(images,imagenames)[:1]:
d = img[0].data;
wh = d<0;
d[wh] = 0;
print "Image %s: replaced %d points"%(name,wh.sum());
updated = True;
if options.transfer:
if len(images) != 2:
parser.error("The --transfer option requires exactly two input images.");
if autoname:
outname = "xfer_" + outname;
print "Transferring %s into coordinate system of %s"%(imagenames[1],imagenames[0]);
images[0][0].data = images[1][0].data;
updated = True;
elif options.diff:
if len(images) != 2:
parser.error("The --diff option requires exactly two input images.");
if autoname:
outname = "diff_" + outname;
print "Computing difference";
data = images[0][0].data;
data -= images[1][0].data;
updated = True;
elif options.mean:
if autoname:
outname = "mean%d_"%len(images) + outname;
print "Computing mean";
data = images[0][0].data;
for img in images[1:]:
data += img[0].data;
data /= len(images);
images = [ images[0] ];
updated = True;
if options.zoom:
z = options.zoom;
if autoname:
outname = "zoom%d_"%z + outname;
if len(images) > 1:
"Too many input images specified for this operation, at most 1 expected";
sys.exit(2);
data = images[0][0].data;
nx = data.shape[-2];
ny = data.shape[-1];
zdata = data[:,:,(nx-z)/2:(nx+z)/2,(ny-z)/2:(ny+z)/2];
#update header
hdr = images[0][0].header
wcs = WCS(hdr, mode="pyfits")
cr1, cr2 = wcs.pix2wcs(ny/2, nx/2)
hdr["CRVAL1"] = cr1
hdr["CRVAL2"] = cr2
hdr["CRPIX1"] = zdata.shape[-1]/2
hdr["CRPIX2"] = zdata.shape[-2]/2
print "Making zoomed image of shape","x".join(map(str,zdata.shape));
images = [ pyfits.PrimaryHDU(zdata, hdr) ];
updated = True;
if options.rescale != 1:
if autoname and not updated:
outname = "rescale_" + outname;
if len(images) > 1:
"Too many input images specified for this operation, at most 1 expected";
sys.exit(2);
print "Applying scaling factor of %f to image values"%options.rescale;
images[0][0].data *= options.rescale;
updated = True;
if updated:
imagenames[0] = outname;
if options.stats:
for ff,filename in zip(images,imagenames):
data = ff[0].data;
min,max,dum1,dum2 = scipy.ndimage.measurements.extrema(data);
sum = data.sum();
mean = sum/data.size;
std = math.sqrt(((data-mean)**2).mean());
print "%s: min %g, max %g, sum %g, np %d, mean %g, std %g"%(filename,min,max,sum,data.size,mean,std);
sys.exit(0);
if updated:
print "Writing output image",outname;
if os.path.exists(outname) and not options.force:
print "Output image exists, rerun with the -f switch to overwrite.";
sys.exit(1);
images[0].writeto(outname,clobber=True);
elif not (options.header or options.stack or options.add_axis or options.reorder):
print "No operations specified. Use --help for help."
shape = None
source = SkyModel.Source(name, pos, flux, shape=shape)
# Adding source peak flux (error) as extra flux attributes for sources,
# and to avoid null values for point sources I_peak = src["Total_flux"]
if shape:
source.setAttribute("I_peak", float(src["peak_flux"]))
source.setAttribute("I_peak_err", float(src["err_peak_flux"]))
else:
source.setAttribute("I_peak", float(src["int_flux"]))
source.setAttribute("I_peak_err", float(src["err_int_flux"]))
return source
data = Table.read('{0}_comp.{1}'.format(
outfile.split('.')[0],
outfile.split('.')[-1]),
format=outfile.split('.')[-1])
for i, src in enumerate(data):
model.sources.append(tigger_src(src, i))
wcs = WCS(image)
centre = wcs.getCentreWCSCoords()
model.ra0, model.dec0 = map(numpy.deg2rad, centre)
model.save(tname_lsm)
# Rename using CORPAT
utils.xrun("tigger-convert", [tname_lsm, "--rename -f"])
def __init__(self, imagename, psfname=None, sourcefinder_name='pybdsm',
saveformat="gaul", makeplots=True, do_psf_corr=True,
do_local_var=True, psf_corr_region=5, local_var_region=10,
rel_excl_src=None, pos_smooth=2, neg_smooth=2, loglevel=0,
thresh_pix=5, thresh_isl=3, neg_thresh_isl=3,
neg_thresh_pix=5, reset_rel=None, prefix=None,
do_nearsources=False, savefits=False,
increase_beam_cluster=False, savemask_pos=False,
savemask_neg=False, no_smooth=True, **kw):
""" Takes in image and extracts sources and makes
reliability estimations..
imagename: Fits image
psfname: PSF fits image, optional.
sourcefinder_name: str, optional. Default 'pybdsm'.
Uses source finder specified.
makeplots: bool, optional. Default is True.
Make reliability plots.
do_psf_corr : bool, optional. Default True.
If True, PSF correlation will be added
as an extra parameter for density estimations.
NB: the PSF fits image must be provided.
do_local_var : bool, optional. Default is True.
If True, adds local variance as an extra parameter,
for density estimations.
do_nearsources: boolean. Default is False.
If true it adds number of nearest neighnours as an extra
parameter. It looks for sources around 5 beam sizes.
psf_corr_region : int, optional. Default value is 5.
Data size to correlate around a source, in beam sizes.
local_var_region: int, optional. Default 10.
Data size to compute the local variance in beam sizes.
rel_excl_src : floats, optional. Default is None.
Excludes sources in a specified region
e.g ra, dec, radius in degrees. For
2 regions: ra1, dec1, radius1: ra2, dec2, radius2, etc.
pos_smooth : float, optional. Default 2.
Masking threshold for the positive image.
For default value 2, data peaks < 2 * image noise
are masked.
neg_smooth : float, optional. Default 2.
Similar to pos_smooth but applied to the negative image.
thresh_isl : float, optional. Default is 3.
Threshold for forming islands in the positive image
thresh_pix : float, optional. Default is 5.
Threshold for model fitting, in positive image.
neg_thresh_isl : float, optional. Default is 3.
Simialr to thresh_isl but for negative image.
neg_thresh_pix : float, optional. Default is 5.
Similar to thresh_pix but for negative image.
savefits: boolean. Default is False.
If True a negative image is saved.
reset_rel: boolean. Default is False. If true then
sources with correlation < 0.002 and rel >0.60
have their reliabilities set to 0.
increase_beam_cluster: boolean, optional. If True, sources
groupings will be increase by 20% the beam size. If False,
the actual beam size will be used. Default is False.
savemask_pos: boolean, optional. If true the mask applied on
the positive side of an image after smoothing is saved.
savemask_neg: Similar to savemask_pos but for the negative
side of an image.
loglevel : int, optional. Default is 0.
Provides Pythonlogging options, 0, 1, 2 and 3 are for info, debug,
error and critial respectively.
kw : kward for source extractions. Should be a mapping e.g
kw['thresh_isl'] = 2.0 or kw['do_polarization'] = True
"""
#
self.smoothing = not no_smooth
self.prefix = prefix
# log level
self.loglevel = loglevel
self.log = utils.logger(self.loglevel, prefix=self.prefix)
# image, psf image
self.imagename = imagename
self.psfname = psfname
with pyfits.open(imagename) as hdu:
self.header = hdu[0].header
self.wcs = WCS(self.header, mode="pyfits")
self.pixelsize = abs(self.header["cdelt1"])
self.bmaj = numpy.deg2rad(self.header["BMAJ"])
# boolean optionals
self.makeplots = makeplots
self.do_local_var = do_local_var
self.nearsources = do_nearsources
self.do_psf_corr = do_psf_corr
self.savemaskpos = savemask_pos
self.savemaskneg = savemask_neg
self.savefits = savefits
self.derel = reset_rel
self.log.info("Catalogues will be saved as %s, where srl is source "
" and gaul is Gaussians. "%saveformat)
self.catalogue_format = "." + saveformat
if not self.psfname:
self.log.info(" No psf provided, do_psf_corr is set to False.")
self.do_psf_corr = False
# computing negative noise
self.noise, self.mean = utils.negative_noise(self.imagename, self.prefix)
self.log.info(" The negative noise is %e Jy/beam"%self.noise)
if self.noise == 0:
self.log.debug(" The negative noise is 0, check image")
# source finder initialization
self.sourcefinder_name = sourcefinder_name
self.log.info(" Using %s source finder to extract the sources."%
self.sourcefinder_name)
self.negimage = self.prefix + "_negative.fits"
utils.invert_image(self.imagename, self.negimage)
# smoothing factors
self.pos_smooth = pos_smooth
self.neg_smooth = neg_smooth
# region to evaluate
self.corrstep = psf_corr_region
self.localstep = local_var_region
self.radiusrm = rel_excl_src
self.do_beam = increase_beam_cluster
beam_pix = int(round(numpy.rad2deg(self.bmaj)/self.pixelsize))
self.locstep = self.localstep * beam_pix
self.cfstep = self.corrstep * beam_pix
self.bmin, self.bpa = self.header["BMIN"], self.header["BPA"]
self.opts_pos = {}
if self.do_beam:
bmaj = self.header["BMAJ"]
self.opts_pos["beam"] = (1.2*bmaj, 1.2*self.bmin, self.bpa)
# Pybdsm or source finder fitting thresholds
self.thresh_isl = thresh_isl
self.thresh_pix = thresh_pix
self.opts_pos = dict(thresh_pix=self.thresh_pix,
thresh_isl=self.thresh_isl)
self.opts_pos.update(kw)
self.opts_neg = {}
self.opts_neg.update(kw)
self.neg_thresh_isl = neg_thresh_isl
self.neg_thresh_pix = neg_thresh_pix
self.opts_neg["thresh_isl"] = self.neg_thresh_isl
self.opts_neg["thresh_pix"] = self.neg_thresh_pix
class load(object):
def __init__(self,
imagename,
psfname=None,
sourcefinder_name='pybdsm',
saveformat="gaul",
makeplots=True,
do_psf_corr=True,
do_local_var=True,
psf_corr_region=5,
local_var_region=10,
rel_excl_src=None,
pos_smooth=2,
neg_smooth=2,
loglevel=0,
thresh_pix=5,
thresh_isl=3,
neg_thresh_isl=3,
neg_thresh_pix=5,
reset_rel=None,
prefix=None,
do_nearsources=False,
savefits=False,
increase_beam_cluster=False,
savemask_pos=False,
savemask_neg=False,
no_smooth=True,
**kw):
""" Takes in image and extracts sources and makes
reliability estimations..
imagename: Fits image
psfname: PSF fits image, optional.
sourcefinder_name: str, optional. Default 'pybdsm'.
Uses source finder specified.
makeplots: bool, optional. Default is True.
Make reliability plots.
do_psf_corr : bool, optional. Default True.
If True, PSF correlation will be added
as an extra parameter for density estimations.
NB: the PSF fits image must be provided.
do_local_var : bool, optional. Default is True.
If True, adds local variance as an extra parameter,
for density estimations.
do_nearsources: boolean. Default is False.
If true it adds number of nearest neighnours as an extra
parameter. It looks for sources around 5 beam sizes.
psf_corr_region : int, optional. Default value is 5.
Data size to correlate around a source, in beam sizes.
local_var_region: int, optional. Default 10.
Data size to compute the local variance in beam sizes.
rel_excl_src : floats, optional. Default is None.
Excludes sources in a specified region
e.g ra, dec, radius in degrees. For
2 regions: ra1, dec1, radius1: ra2, dec2, radius2, etc.
pos_smooth : float, optional. Default 2.
Masking threshold for the positive image.
For default value 2, data peaks < 2 * image noise
are masked.
neg_smooth : float, optional. Default 2.
Similar to pos_smooth but applied to the negative image.
thresh_isl : float, optional. Default is 3.
Threshold for forming islands in the positive image
thresh_pix : float, optional. Default is 5.
Threshold for model fitting, in positive image.
neg_thresh_isl : float, optional. Default is 3.
Simialr to thresh_isl but for negative image.
neg_thresh_pix : float, optional. Default is 5.
Similar to thresh_pix but for negative image.
savefits: boolean. Default is False.
If True a negative image is saved.
reset_rel: boolean. Default is False. If true then
sources with correlation < 0.002 and rel >0.60
have their reliabilities set to 0.
increase_beam_cluster: boolean, optional. If True, sources
groupings will be increase by 20% the beam size. If False,
the actual beam size will be used. Default is False.
savemask_pos: boolean, optional. If true the mask applied on
the positive side of an image after smoothing is saved.
savemask_neg: Similar to savemask_pos but for the negative
side of an image.
loglevel : int, optional. Default is 0.
Provides Pythonlogging options, 0, 1, 2 and 3 are for info, debug,
error and critial respectively.
kw : kward for source extractions. Should be a mapping e.g
kw['thresh_isl'] = 2.0 or kw['do_polarization'] = True
"""
#
self.smoothing = not no_smooth
self.prefix = prefix
# log level
self.loglevel = loglevel
self.log = utils.logger(self.loglevel, prefix=self.prefix)
# image, psf image
self.imagename = imagename
self.psfname = psfname
with pyfits.open(imagename) as hdu:
self.header = hdu[0].header
self.wcs = WCS(self.header, mode="pyfits")
self.pixelsize = abs(self.header["cdelt1"])
self.bmaj = numpy.deg2rad(self.header["BMAJ"])
# boolean optionals
self.makeplots = makeplots
self.do_local_var = do_local_var
self.nearsources = do_nearsources
self.do_psf_corr = do_psf_corr
self.savemaskpos = savemask_pos
self.savemaskneg = savemask_neg
self.savefits = savefits
self.derel = reset_rel
self.log.info("Catalogues will be saved as %s, where srl is source "
" and gaul is Gaussians. " % saveformat)
self.catalogue_format = "." + saveformat
if not self.psfname:
self.log.info(" No psf provided, do_psf_corr is set to False.")
self.do_psf_corr = False
# computing negative noise
self.noise, self.mean = utils.negative_noise(self.imagename,
self.prefix)
self.log.info(" The negative noise is %e Jy/beam" % self.noise)
if self.noise == 0:
self.log.debug(" The negative noise is 0, check image")
# source finder initialization
self.sourcefinder_name = sourcefinder_name
self.log.info(" Using %s source finder to extract the sources." %
self.sourcefinder_name)
self.negimage = self.prefix + "_negative.fits"
utils.invert_image(self.imagename, self.negimage)
# smoothing factors
self.pos_smooth = pos_smooth
self.neg_smooth = neg_smooth
# region to evaluate
self.corrstep = psf_corr_region
self.localstep = local_var_region
self.radiusrm = rel_excl_src
self.do_beam = increase_beam_cluster
beam_pix = int(round(numpy.rad2deg(self.bmaj) / self.pixelsize))
self.locstep = self.localstep * beam_pix
self.cfstep = self.corrstep * beam_pix
self.bmin, self.bpa = self.header["BMIN"], self.header["BPA"]
self.opts_pos = {}
if self.do_beam:
bmaj = self.header["BMAJ"]
self.opts_pos["beam"] = (1.2 * bmaj, 1.2 * self.bmin, self.bpa)
# Pybdsm or source finder fitting thresholds
self.thresh_isl = thresh_isl
self.thresh_pix = thresh_pix
self.opts_pos = dict(thresh_pix=self.thresh_pix,
thresh_isl=self.thresh_isl)
self.opts_pos.update(kw)
self.opts_neg = {}
self.opts_neg.update(kw)
self.neg_thresh_isl = neg_thresh_isl
self.neg_thresh_pix = neg_thresh_pix
self.opts_neg["thresh_isl"] = self.neg_thresh_isl
self.opts_neg["thresh_pix"] = self.neg_thresh_pix
def source_finder(self,
image=None,
thresh=None,
prefix=None,
noise=None,
output=None,
savemask=None,
**kw):
#kw.update(kwards)
tpos = None
naxis = self.header["NAXIS1"]
boundary = numpy.array([self.locstep, self.cfstep])
#trim_box = (boundary.max(), naxis - boundary.max(),
# boundary.max(), naxis - boundary.max())
trim_box = None
# data smoothing
if self.smoothing:
ext = utils.fits_ext(image)
tpos = tempfile.NamedTemporaryFile(suffix="." + ext, dir=".")
tpos.flush()
mask, noise = utils.thresh_mask(image,
tpos.name,
thresh=thresh,
noise=self.noise,
sigma=True,
smooth=True,
prefix=prefix,
savemask=savemask)
# using the masked image for forming islands
kw["detection_image"] = tpos.name
kw["blank_limit"] = self.noise / 1.0e5
# source extraction
utils.sources_extraction(image=image,
output=output,
sourcefinder_name=self.sourcefinder_name,
trim_box=trim_box,
prefix=self.prefix,
**kw)
if tpos:
tpos.close()
def remove_sources_within(self, model):
sources = model.sources
rel_remove = self.radiusrm[0].split(":")
for i in range(len(rel_remove)):
ra, dec, tolerance = rel_remove[i].split(",")
ra_r = numpy.deg2rad(float(ra))
dec_r = numpy.deg2rad(float(dec))
tolerance_r = numpy.deg2rad(float(tolerance))
within = model.getSourcesNear(ra_r, dec_r, tolerance_r)
for src in sorted(sources):
if src in within:
model.sources.remove(src)
return model
def params(self, modelfits):
# reads in source finder output
with pyfits.open(modelfits) as hdu:
data = hdu[1].data
tfile = tempfile.NamedTemporaryFile(suffix=".txt")
tfile.flush()
# writes a catalogue in a temporaty txt file
with open(tfile.name, "w") as std:
std.write("#format:name ra_rad dec_rad i emaj_r emin_r pa_r\n")
model = Tigger.load(tfile.name) # open a tmp. file
peak, total, area, loc, corr = [], [], [], [], []
for i in range(len(data)):
flux = data["Total_flux"][i]
dc_emaj, dc_emin = data["DC_Maj"][i], data["DC_Min"][i]
ra, dec = data["RA"][i], data["DEC"][i]
pa = data["DC_PA"][i]
name = "SRC%d" % i
peak_flux = data["Peak_flux"][i]
posrd = ModelClasses.Position(numpy.deg2rad(ra),
numpy.deg2rad(dec))
flux_I = ModelClasses.Polarization(flux, 0, 0, 0)
if dc_emaj == 0 and dc_emin == 0:
shape = None
else:
shape = ModelClasses.Gaussian(numpy.deg2rad(dc_emaj),
numpy.deg2rad(dc_emin),
numpy.deg2rad(pa))
srs = SkyModel.Source(name, posrd, flux_I, shape=shape)
# using convolved maj and min for reliability estimate
emaj, emin = data["Maj"][i], data["Min"][i]
# area: find ex and ey if are 0 assign beam size
if emaj or emin == 0:
srcarea = math.pi * (numpy.rad2deg(self.bmaj)) * pow(3600.0, 2) *\
(numpy.rad2deg(self.bmin))
if emaj and emin > 0:
srcarea = emaj * emin * math.pi * pow(3600.0, 2) # arcsecond
# only accepts sources with flux > 0 and not nan RA and DEC
# and local variance
pos = [self.wcs.wcs2pix(*(ra, dec))
][0] #positions from deg to pixel
with pyfits.open(self.negimage) as hdu:
negdata = utils.image_data(hdu[0].data)
if flux > 0 and peak_flux > 0 and not math.isnan(float(ra))\
and not math.isnan(float(dec)):
local = utils.compute_local_variance(negdata, pos,
self.locstep)
srs.setAttribute("local_variance", local)
if not math.isnan(float(local)) or local > 0:
if self.psfname:
pdata, psf = utils.compute_psf_correlation(
self.imagename, self.psfname, pos, self.cfstep)
if len(pdata) == len(psf):
c_region = numpy.corrcoef((pdata, psf))
cf = (numpy.diag((numpy.rot90(c_region))**
2).sum())**0.5 / 2**0.5
srs.setAttribute("correlation_factor", cf)
corr.append(cf)
model.sources.append(srs)
peak.append(peak_flux)
total.append(flux)
area.append(srcarea)
loc.append(local)
else:
model.sources.append(srs)
peak.append(peak_flux)
total.append(flux)
area.append(srcarea)
loc.append(local)
labels = dict(size=(0, "Log$_{10}$(Source area)"),
peak=(1, "Log$_{10}$( Peak flux [Jy] )"),
tot=(2, "Log$_{10}$( Total flux [Jy] )"))
if self.do_psf_corr:
labels.update({"coeff": (len(labels), "Log$_{10}$ (CF)")})
if self.do_local_var:
labels.update(
{"local": (len(labels), "Log$_{10}$(Local Variance)")})
if self.nearsources:
labels.update({"near": (len(labels), "Log$_{10}$(Near Sources)")})
nsrc = len(model.sources)
out = numpy.zeros([nsrc, len(labels)])
# returning parameters
for i, src in enumerate(model.sources):
ra, dec = src.pos.ra, src.pos.dec
near = model.getSourcesNear(ra, dec, 5 * self.bmaj)
nonear = len(near)
if self.nearsources:
src.setAttribute("neibours", nonear)
if self.do_psf_corr and self.do_local_var and self.nearsources:
out[i,
...] = area[i], peak[i], total[i], corr[i], loc[i], nonear
elif self.do_psf_corr and self.do_local_var and not self.nearsources:
out[i, ...] = area[i], peak[i], total[i], corr[i], loc[i]
elif self.do_psf_corr and self.nearsources and not self.do_local_var:
out[i, ...] = area[i], peak[i], total[i], corr[i], nonear
elif not self.do_psf_corr and self.do_local_var and self.nearsources:
out[i, ...] = area[i], peak[i], total[i], loc[i], nonear
elif self.do_psf_corr and not self.do_local_var and not self.nearsources:
out[i, ...] = area[i], peak[i], total[i], corr[i]
elif not self.do_psf_corr and self.do_local_var and not self.nearsources:
out[i, ...] = area[i], peak[i], total[i], loc[i]
elif not self.do_psf_corr and not self.do_local_var and self.nearsources:
out[i, ...] = area[i], peak[i], total[i], nonear
else:
out[i, ...] = area[i], peak[i], total[i]
# removes the rows with 0s
removezeros = (out == 0).sum(1)
output = out[removezeros <= 0, :]
return model, numpy.log10(output), labels
def get_reliability(self):
# finding sources
self.log.info(" Extracting the sources on both sides ")
pfile = self.prefix + self.catalogue_format + ".fits"
nfile = self.prefix + "_negative" + self.catalogue_format + ".fits"
# i need to catch mmap.mmap error here
# running a source finder
self.source_finder(image=self.negimage,
output=nfile,
thresh=self.neg_smooth,
savemask=self.savemaskneg,
prefix=self.prefix,
**self.opts_neg)
self.source_finder(image=self.imagename,
output=pfile,
thresh=self.pos_smooth,
savemask=self.savemaskpos,
prefix=self.prefix,
**self.opts_pos)
self.log.info(" Source Finder completed successfully ")
pmodel, positive, labels = self.params(pfile)
nmodel, negative, labels = self.params(nfile)
# setting up a kernel, Gaussian kernel
bandwidth = []
for plane in negative.T:
bandwidth.append(plane.std())
nplanes = len(labels)
cov = numpy.zeros([nplanes, nplanes])
nnsrc = len(negative)
npsrc = len(positive)
self.log.info(" There are %d positive and %d negtive detections " %
(npsrc, nnsrc))
if nnsrc == 0 or npsrc == 0:
self.log.error(
"The resulting array has length of 0 thus cannot compute"
" the reliability. Aborting.")
self.log.info(" Computing the reliabilities ")
for i in range(nplanes):
for j in range(nplanes):
if i == j:
cov[i, j] = bandwidth[i] * ((4.0 / (
(nplanes + 2) * nnsrc))**(1.0 / (nplanes + 4.0)))
self.log.info("The resulting covariance matrix is %r" % cov)
pcov = utils.gaussian_kde_set_covariance(positive.T, cov)
ncov = utils.gaussian_kde_set_covariance(negative.T, cov)
# get number densities
nps = pcov(positive.T) * npsrc
nns = ncov(positive.T) * nnsrc
# define reliability of positive catalog
rel = (nps - nns) / nps
for src, rf in zip(pmodel.sources, rel):
src.setAttribute("rel", rf)
self.log.info(" Saved the reliabilities values.")
# remove sources with poor correlation and high reliability,
# the values are currently arbitrary
if self.do_psf_corr and self.derel:
for s in pmodel.sources:
cf, r = s.correlation_factor, s.rel
if cf < 0.006 and r > 0.60:
s.rel = 0.0
if self.makeplots:
savefig = self.prefix + "_planes.png"
utils.plot(positive,
negative,
rel=rel,
labels=labels,
savefig=savefig,
prefix=self.prefix)
# removes sources in a given radius from the phase center
if self.radiusrm:
self.log.info(" Remove sources ra, dec, radius of %r"
" from the phase center" % self.radiusrm)
pmodel = self.remove_sources_within(pmodel)
if not self.savefits:
self.log.info(" Deleting the negative image.")
os.system("rm -r %s" % self.negimage)
# Set field Center
pmodel.ra0, pmodel.dec0 = map(numpy.deg2rad,
self.wcs.getCentreWCSCoords())
return pmodel, nmodel, self.locstep
def accept(self):
"""Tries to make a brick, and closes the dialog if successful."""
sources = [
src for src in self.model.sources
if src.selected and src.typecode == 'pnt'
]
filename = self.wfile.filename()
if not self._fileSelected(filename):
return
# get PB expression
pbfunc = None
if self.wpb_apply.isChecked():
pbexp = str(self.wpb_exp.text())
try:
pbfunc = eval("lambda r,fq:" + pbexp)
except Exception as err:
QMessageBox.warning(
self, "Error parsing PB experssion",
"Error parsing primary beam expression %s: %s" %
(pbexp, str(err)))
return
# get frequency
freq = str(self.wfreq.text())
freq = float(freq) * 1e+6 if freq else None
# get pad factor
pad = str(self.wpad.text())
pad = max(float(pad), 1) if pad else 1
# read fits file
busy = BusyIndicator()
try:
input_hdu = pyfits.open(filename)[0]
except Exception as err:
busy = None
QMessageBox.warning(
self, "Error reading FITS",
"Error reading FITS file %s: %s" % (filename, str(err)))
return
# reset data if asked to
if self.woverwrite.isChecked():
input_hdu.data[...] = 0
# insert sources
Imaging.restoreSources(input_hdu,
sources,
0,
primary_beam=pbfunc,
freq=freq)
# save fits file
try:
# pyfits seems to produce an exception:
# TypeError: formatwarning() takes exactly 4 arguments (5 given)
# when attempting to overwrite a file. As a workaround, remove the file first.
if os.path.exists(filename):
os.remove(filename)
input_hdu.writeto(filename)
except Exception as err:
traceback.print_exc()
busy = None
QMessageBox.warning(
self, "Error writing FITS",
"Error writing FITS file %s: %s" % (filename, str(err)))
return
changed = False
sources = self.model.sources
# remove sources from model if asked to
if self.wdel.isChecked():
sources = [
src for src in sources
if not (src.selected and src.typecode == 'pnt')
]
changed = True
# add image to model if asked to
if self.wadd.isChecked():
hdr = input_hdu.header
# get image parameters
max_flux = float(input_hdu.data.max())
wcs = WCS(hdr, mode='pyfits')
# Get reference pixel coordinates
# wcs.getCentreWCSCoords() doesn't work, as that gives us the middle of the image
# So scan the header to get the CRPIX values
ra0 = dec0 = 1
for iaxis in range(hdr['NAXIS']):
axs = str(iaxis + 1)
name = hdr.get('CTYPE' + axs, axs).upper()
if name.startswith("RA"):
ra0 = hdr.get('CRPIX' + axs, 1) - 1
elif name.startswith("DEC"):
dec0 = hdr.get('CRPIX' + axs, 1) - 1
# convert pixel to degrees
ra0, dec0 = wcs.pix2wcs(ra0, dec0)
ra0 *= DEG
dec0 *= DEG
sx, sy = wcs.getHalfSizeDeg()
sx *= DEG
sy *= DEG
nx, ny = input_hdu.data.shape[-1:-3:-1]
# check if this image is already contained in the model
for src in sources:
if isinstance(getattr(src, 'shape', None),
ModelClasses.FITSImage) and os.path.samefile(
src.shape.filename, filename):
# update source parameters
src.pos.ra, src.pos.dec = ra0, dec0
src.flux.I = max_flux
src.shape.ex, src.shape.ey = sx, sy
src.shape.nx, src.shape.ny = nx, ny
src.shape.pad = pad
break
# not contained, make new source object
else:
pos = ModelClasses.Position(ra0, dec0)
flux = ModelClasses.Flux(max_flux)
shape = ModelClasses.FITSImage(sx,
sy,
0,
os.path.basename(filename),
nx,
ny,
pad=pad)
img_src = SkyModel.Source(os.path.splitext(
os.path.basename(filename))[0],
pos,
flux,
shape=shape)
sources.append(img_src)
changed = True
if changed:
self.model.setSources(sources)
self.model.emitUpdate(SkyModel.SkyModel.UpdateAll, origin=self)
self.parent().showMessage("Wrote %d sources to FITS file %s" %
(len(sources), filename))
busy = None
return QDialog.accept(self)
def __init__(self,
imagename,
psfname=None,
sourcefinder_name='pybdsm',
saveformat="gaul",
makeplots=True,
do_psf_corr=True,
do_local_var=True,
psf_corr_region=5,
local_var_region=10,
rel_excl_src=None,
pos_smooth=2,
neg_smooth=2,
loglevel=0,
thresh_pix=5,
thresh_isl=3,
neg_thresh_isl=3,
neg_thresh_pix=5,
reset_rel=None,
prefix=None,
do_nearsources=False,
savefits=False,
increase_beam_cluster=False,
savemask_pos=False,
savemask_neg=False,
no_smooth=True,
**kw):
""" Takes in image and extracts sources and makes
reliability estimations..
imagename: Fits image
psfname: PSF fits image, optional.
sourcefinder_name: str, optional. Default 'pybdsm'.
Uses source finder specified.
makeplots: bool, optional. Default is True.
Make reliability plots.
do_psf_corr : bool, optional. Default True.
If True, PSF correlation will be added
as an extra parameter for density estimations.
NB: the PSF fits image must be provided.
do_local_var : bool, optional. Default is True.
If True, adds local variance as an extra parameter,
for density estimations.
do_nearsources: boolean. Default is False.
If true it adds number of nearest neighnours as an extra
parameter. It looks for sources around 5 beam sizes.
psf_corr_region : int, optional. Default value is 5.
Data size to correlate around a source, in beam sizes.
local_var_region: int, optional. Default 10.
Data size to compute the local variance in beam sizes.
rel_excl_src : floats, optional. Default is None.
Excludes sources in a specified region
e.g ra, dec, radius in degrees. For
2 regions: ra1, dec1, radius1: ra2, dec2, radius2, etc.
pos_smooth : float, optional. Default 2.
Masking threshold for the positive image.
For default value 2, data peaks < 2 * image noise
are masked.
neg_smooth : float, optional. Default 2.
Similar to pos_smooth but applied to the negative image.
thresh_isl : float, optional. Default is 3.
Threshold for forming islands in the positive image
thresh_pix : float, optional. Default is 5.
Threshold for model fitting, in positive image.
neg_thresh_isl : float, optional. Default is 3.
Simialr to thresh_isl but for negative image.
neg_thresh_pix : float, optional. Default is 5.
Similar to thresh_pix but for negative image.
savefits: boolean. Default is False.
If True a negative image is saved.
reset_rel: boolean. Default is False. If true then
sources with correlation < 0.002 and rel >0.60
have their reliabilities set to 0.
increase_beam_cluster: boolean, optional. If True, sources
groupings will be increase by 20% the beam size. If False,
the actual beam size will be used. Default is False.
savemask_pos: boolean, optional. If true the mask applied on
the positive side of an image after smoothing is saved.
savemask_neg: Similar to savemask_pos but for the negative
side of an image.
loglevel : int, optional. Default is 0.
Provides Pythonlogging options, 0, 1, 2 and 3 are for info, debug,
error and critial respectively.
kw : kward for source extractions. Should be a mapping e.g
kw['thresh_isl'] = 2.0 or kw['do_polarization'] = True
"""
#
self.smoothing = not no_smooth
self.prefix = prefix
# log level
self.loglevel = loglevel
self.log = utils.logger(self.loglevel, prefix=self.prefix)
# image, psf image
self.imagename = imagename
self.psfname = psfname
with pyfits.open(imagename) as hdu:
self.header = hdu[0].header
self.wcs = WCS(self.header, mode="pyfits")
self.pixelsize = abs(self.header["cdelt1"])
self.bmaj = numpy.deg2rad(self.header["BMAJ"])
# boolean optionals
self.makeplots = makeplots
self.do_local_var = do_local_var
self.nearsources = do_nearsources
self.do_psf_corr = do_psf_corr
self.savemaskpos = savemask_pos
self.savemaskneg = savemask_neg
self.savefits = savefits
self.derel = reset_rel
self.log.info("Catalogues will be saved as %s, where srl is source "
" and gaul is Gaussians. " % saveformat)
self.catalogue_format = "." + saveformat
if not self.psfname:
self.log.info(" No psf provided, do_psf_corr is set to False.")
self.do_psf_corr = False
# computing negative noise
self.noise, self.mean = utils.negative_noise(self.imagename,
self.prefix)
self.log.info(" The negative noise is %e Jy/beam" % self.noise)
if self.noise == 0:
self.log.debug(" The negative noise is 0, check image")
# source finder initialization
self.sourcefinder_name = sourcefinder_name
self.log.info(" Using %s source finder to extract the sources." %
self.sourcefinder_name)
self.negimage = self.prefix + "_negative.fits"
utils.invert_image(self.imagename, self.negimage)
# smoothing factors
self.pos_smooth = pos_smooth
self.neg_smooth = neg_smooth
# region to evaluate
self.corrstep = psf_corr_region
self.localstep = local_var_region
self.radiusrm = rel_excl_src
self.do_beam = increase_beam_cluster
beam_pix = int(round(numpy.rad2deg(self.bmaj) / self.pixelsize))
self.locstep = self.localstep * beam_pix
self.cfstep = self.corrstep * beam_pix
self.bmin, self.bpa = self.header["BMIN"], self.header["BPA"]
self.opts_pos = {}
if self.do_beam:
bmaj = self.header["BMAJ"]
self.opts_pos["beam"] = (1.2 * bmaj, 1.2 * self.bmin, self.bpa)
# Pybdsm or source finder fitting thresholds
self.thresh_isl = thresh_isl
self.thresh_pix = thresh_pix
self.opts_pos = dict(thresh_pix=self.thresh_pix,
thresh_isl=self.thresh_isl)
self.opts_pos.update(kw)
self.opts_neg = {}
self.opts_neg.update(kw)
self.neg_thresh_isl = neg_thresh_isl
self.neg_thresh_pix = neg_thresh_pix
self.opts_neg["thresh_isl"] = self.neg_thresh_isl
self.opts_neg["thresh_pix"] = self.neg_thresh_pix
