def bdsm(params):
'''
This runs PyBDSM on the image to make a mask from the Gaussian output from the source
finder.
'''
# Export the Image as Fits
fits = mirexec.TaskFits()
fits.in_ = params.image
fits.op = 'xyout'
fits.out = params.image + '.fits'
fits.snarf()
# Run PyBDSM on the Image
img = bdsm.process_image(params.image + '.fits')
img.export_image(outfile=params.image + 'gaus_model.fits',
im_type='gaus_model')
img.write_catalog(outfile=params.image + 'gaus_model.txt')
# Use the Gaussian Image File as a Model
fits.in_ = params.image + 'gaus_model.fits'
fits.op = 'xyin'
fits.out = params.lsm
fits.snarf()
params.model = params.lsm
#print params
sys.exit(0)
tout = selfcal(params)
def sources_extraction(image, output=None,
sourcefinder_name="pybdsm",
prefix=None, **kw):
"""Runs pybdsm on the specified 'image', converts the
results into a Tigger model and writes it to output.
image : Fits image data
ou
A Catalog name to store the extracted sources
"""
# start with default PYBDSM options
opts = {}
opts.update(kw)
catalogue_format = output.split(".")[-2]
log = logger(level=0, prefix=prefix)
log.info(" Source finding begins...")
if sourcefinder_name.lower() == "pybdsm":
from lofar import bdsm
img = bdsm.process_image(image, group_by_isl=True, **kw)
img.write_catalog(outfile=output, format="fits",
catalog_type=catalogue_format, clobber=True)
log.info(" Source finding was succesfully performed.")
return output
def do_makecleanmask_field(image_name,threshpix,threshisl,atrousdo,ncores=8):
mask_name = image_name.split('.image')[0] + '.cleanmask'
gausmodel = image_name.split('.image')[0] + '.gausmodel'
logging.info('makecleanmask_field: Making mask: '+mask_name)
if atrousdo:
threshisl = 3.0
logging.info('Changing island threshold to 3 because atrous_do=True')
os.system('rm -rf ' + mask_name)
os.system('rm -rf ' + gausmodel)
os.system('cp -r ' + image_name + ' ' + mask_name)
# DO THE SOURCE DETECTION
img = bdsm.process_image(image_name, mean_map='zero',
rms_box=(300,60),
thresh_pix=threshpix,
thresh_isl=threshisl,
atrous_do=atrousdo,ini_method='curvature',
adaptive_rms_box=True,
adaptive_thresh=150,
rms_box_bright=(70,10),ncores=ncores)
#img.show_fit()
# WRITE THE GAUSSIAN MODEL FITS
#img.export_image(img_type='gaus_model', outfile=gausmodel)
img.export_image(img_type='island_mask',img_format='casa',outfile=mask_name, mask_dilation=2, clobber=True)
def bdsm(params): ''' This runs PyBDSM on the image to make a mask from the Gaussian output from the source finder. ''' # Export the Image as Fits fits = mirexec.TaskFits() fits.in_ = params.image fits.op = 'xyout' fits.out = params.image+'.fits' fits.snarf() # Run PyBDSM on the Image img = bdsm.process_image(params.image+'.fits') img.export_image(outfile = params.image+'gaus_model.fits', im_type='gaus_model') img.write_catalog(outfile = params.image+'gaus_model.txt') # Use the Gaussian Image File as a Model fits.in_ = params.image+'gaus_model.fits' fits.op = 'xyin' fits.out = params.lsm fits.snarf() params.model = params.lsm #print params sys.exit(0) tout = selfcal(params)
def do_makecleanmask_field_wsclean(image_name,threshpix,threshisl,atrousdo,ncores=8):
mask_name = image_name.split('-image')[0] + '.fitsmask'
logging.info('makecleanmask_field_wsclean: Making mask: '+mask_name)
if atrousdo:
threshisl = 3.0
logging.info('Changing island threshold to %.1f because atrous_do=True' % threshisl)
os.system('rm -rf ' + mask_name)
# DO THE SOURCE DETECTION
img = bdsm.process_image(image_name, mean_map='zero',
rms_box=(300,60), thresh_pix=threshpix,
thresh_isl=threshisl,
atrous_do=atrousdo,
adaptive_rms_box=True,
adaptive_thresh=150,
rms_box_bright=(70,10),
atrous_jmax=3,ncores=ncores)
#img.show_fit()
# WRITE THE MASK FITS
img.export_image(img_type='island_mask', img_format='fits', outfile=mask_name)
def selfCalRunFuncSrcExtraction(self):
if self.i < self.nbCycle:
#extract the source model with pybdsm
print ''
print 'extraction by pybdsm: bdsm.process_image %s,adaptive_rms_box=True,advanced_opts=True,detection_image=%s,thresh_isl=%s,thresh_pix=%s,rms_box=(%s,%s),blank_limit=1E-4,atrous_do=True'%("""%sImage_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i],self.i),"""%sImage_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i],self.i),self.thresh_isl,self.thresh_pix,self.RMS_BOX[0],self.RMS_BOX[1])
print ''
#extract the source model with pybdsm
img = bdsm.process_image("""%sImage_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i],self.i),adaptive_rms_box='True',advanced_opts='True',detection_image="""%sImage_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i],self.i),thresh_isl='%s'%(self.thresh_isl),thresh_pix='%s'%(self.thresh_pix),rms_box=(self.RMS_BOX[0],self.RMS_BOX[1]),blank_limit=1E-4,atrous_do='True')
#write bbs catalog
img.write_catalog(outfile="""%sSkymodel_Iter%s"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='bbs',correct_proj='True')
#write ds9 catalog
img.write_catalog(outfile="""%sSkymodel_Iter%s.reg"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='ds9',correct_proj='True')
#write fits catalog
img.write_catalog(outfile="""%sSkymodel_Iter%s.fits"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='fits',correct_proj='True')
if self.i == self.nbCycle:
#extract the source model with pybdsm
print ''
print 'extraction by pybdsm: bdsm.process_image %s,adaptive_rms_box=True,advanced_opts=True,detection_image=%s,thresh_isl=%s,thresh_pix=%s,rms_box=(%s,%s),blank_limit=1E-4,atrous_do=True'%("""%sFinal_Image_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),"""%sFinal_Image_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),self.thresh_isl,self.thresh_pix,self.RMS_BOX[0],self.RMS_BOX[1])
print ''
#extract the source model with pybdsm
img = bdsm.process_image("""%sFinal_Image_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),adaptive_rms_box='True',advanced_opts='True',detection_image="""%sFinal_Image_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),thresh_isl='%s'%(self.thresh_isl),thresh_pix='%s'%(self.thresh_pix),rms_box=(self.RMS_BOX[0],self.RMS_BOX[1]),blank_limit=1E-4,atrous_do='True')
#write bbs catalog
img.write_catalog(outfile="""%sFinal_Skymodel_Iter%s"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='bbs',correct_proj='True')
#write ds9 catalog
img.write_catalog(outfile="""%sFinal_Skymodel_Iter%s.reg"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='ds9',correct_proj='True')
#write fits catalog
img.write_catalog(outfile="""%sFinal_Skymodel_Iter%s.fits"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='fits',correct_proj='True')
def sources_extraction(image, output=None,
sourcefinder_name="pybdsm",
prefix=None, **kw):
"""Runs pybdsm on the specified 'image', converts the
results into a Tigger model and writes it to output.
image : Fits image data
output : Tigger format, default image name.lsm.html
A Catalog name to store the extracted sources
"""
ext = fits_ext(image)
output = output or image.replace(ext, ".lsm.html")
gaul = output + ".gaul"
# start with default PYBDSM options
opts = {}
opts.update(kw)
log = logger(level=0, prefix=prefix)
if sourcefinder_name.lower() == "pybdsm":
from lofar import bdsm
img = bdsm.process_image(image, group_by_isl=True, **kw)
img.write_catalog(outfile=gaul, format="ascii",
catalog_type="gaul", clobber=True)
# converting the model to Tigger
tc = ["tigger-convert", gaul, output,
"-t","Gaul","-f","--rename","-o","Tigger"]
process = subprocess.Popen([" ".join(["%s"%item for item in tc])],
stderr = subprocess.PIPE if not isinstance(sys.stderr,
file) else sys.stderr,
stdout = subprocess.PIPE if not isinstance(sys.stdout,
file) else sys.stdout,
shell=True)
if process.stdout or process.stderr:
out,err = process.comunicate()
sys.stdout.write(out)
sys.stderr.write(err)
out = None
else:
process.wait()
if process.returncode:
log.error("tigger-convert returns errr code %d"%
(process.returncode))
else:
log.info("DONE: tigger-convert succeeded catalog is %s"%output)
verifyModel(output)
def obsPreprocessSrcExtractionFunc(self):
#extract the source model with pybdsm
print ''
print 'extraction by pybdsm: bdsm.process_image %s,adaptive_rms_box=True,advanced_opts=True,detection_image=%s,thresh_isl=%s,thresh_pix=%s,rms_box=(%s,%s),rms_box_bright=(%s,%s),adaptive_thresh=30,blank_limit=1E-4,atrous_do=True,ini_method=curvature,psf_vary_do=True,psf_stype_only=False,psf_snrcut=5,psf_snrcutstack=5,psf_snrtop=0.3' % (
'%sImage_substraction%s.restored.corr' %
(self.preprocessImageDir, self.i),
'%sImage_substraction%s.restored' %
(self.preprocessImageDir, self.i), self.thresh_isl,
self.thresh_pix, 80, 10, 40, 10)
print ''
#extract the source model with pybdsm
img = bdsm.process_image(
'%sImage_substraction%s.restored.corr' %
(self.preprocessImageDir, self.i),
adaptive_rms_box='True',
advanced_opts='True',
detection_image='%sImage_substraction%s.restored' %
(self.preprocessImageDir, self.i),
thresh_isl='%s' % (self.thresh_isl),
thresh_pix='%s' % (self.thresh_pix),
rms_box=(80, 10),
rms_box_bright=(40, 10),
adaptive_thresh=30,
blank_limit=1E-4,
atrous_do='True',
ini_method='curvature'
) #,psf_vary_do='True',psf_stype_only='False',psf_snrcut=5,psf_snrcutstack=5,psf_snrtop=0.3)
#write bbs catalog
img.write_catalog(outfile="""%sSkymodel_substraction%s""" %
(self.preprocessSkymodelDir, self.i),
catalog_type='gaul',
format='bbs',
correct_proj='True',
srcroot='source')
#write ds9 catalog
img.write_catalog(outfile="""%sSkymodel_substraction%s.reg""" %
(self.preprocessSkymodelDir, self.i),
catalog_type='gaul',
format='ds9',
correct_proj='True',
srcroot='source')
#write fits catalog
img.write_catalog(outfile="""%sSkymodel_substraction%s.fits""" %
(self.preprocessSkymodelDir, self.i),
catalog_type='gaul',
format='fits',
correct_proj='True',
srcroot='source')
def docat(infile):
print 'Making catalogue for',infile
txtout=infile+'.catalog'
fitsout=infile+'.catalog.fits'
smout=infile+'.skymodel'
if os.path.isfile(smout):
print 'Catalogue already exists!'
else:
img=bdsm.process_image(infile,thresh_pix=5,fix_to_beam=True,rms_box=(55,12), adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(80,20),mean_map='zero')
img.write_catalog(outfile=txtout,clobber='True',format='ascii')
img.write_catalog(outfile=smout,clobber='True',format='bbs',catalog_type='gaul')
img.write_catalog(outfile=fitsout,clobber='True',format='fits',catalog_type='gaul')
def main(image_name, mask_name, image_beam=False, atrous_do=False, threshisl=0.0, threshpix=0.0, rmsbox=None,
iterate_threshold=False, adaptive_rmsbox=False, beam=None, img_format=None):
convbox = rmsbox.rstrip(')').lstrip('(').split(',')
rmsbox = (int(convbox[0]), int(convbox[1]))
if image_beam:
print 'do stuff for the beam'
if atrous_do:
threshisl = 4.0
if iterate_threshold:
# Start with high threshold and lower it until we get at least one island
threshpix_orig = threshpix
threshisl_orig = threshisl
nisl = 0
threshpix = 25
threshisl = 15
while nisl == 0:
img = bdsm.process_image(image_name, mean_map='zero', rms_box=rmsbox,
thresh_pix=numpy.float(threshpix), thresh_isl=numpy.float(threshisl),
atrous_do=atrous_do, ini_method='curvature', beam=beam,
adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=20, quiet=True)
nisl = img.nisl
threshpix /= 1.2
threshisl /= 1.2
threshpix = threshpix_orig
threshisl = threshisl_orig
else:
img = bdsm.process_image(image_name, mean_map='zero', rms_box=rmsbox,
thresh_pix=numpy.float(threshpix), thresh_isl=numpy.float(threshisl),
atrous_do=atrous_do, ini_method='curvature', beam=beam,
adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=20, quiet=True)
img.export_image(img_type='island_mask', mask_dilation=0, outfile=mask_name,
img_format=img_format, clobber=True)
log_file = mask_name + '.log'
with open(log_file, 'wb') as f:
f.write('# 5-sigma clipped rms (Jy/beam): {0}'.format(5.0 * img.clipped_rms))
def make_mask(image_name,
threshpix=5,
threshisl=3,
atrous_do=False,
mask_name=None,
rmsbox=(55, 12),
mask_combine=None):
import sys, os
import numpy as np
import astropy.io.fits as pyfits
import casacore
import lofar.bdsm as bdsm
# wavelets are required to fit gaussians
if atrous_do: stop_at = None
else: stop_at = 'isl'
# DO THE SOURCE DETECTION
img = bdsm.process_image(image_name, mean_map='zero', rms_box=rmsbox, \
thresh_pix=int(threshpix), thresh_isl=int(threshisl), atrous_do=atrous_do, atrous_jmax=3, \
adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(23,10), \
stop_at=stop_at, blank_limit=1e-5, quiet=True)
# WRITE THE MASK FITS
if mask_name == None: mask_name = image_name + '.newmask'
print 'Making mask:', mask_name
img.export_image(img_type='island_mask',
img_format='casa',
outfile=mask_name,
clobber=True)
del img
# do an pixel-by-pixel "OR" operation with a given mask
if mask_combine != None:
print "Combining with " + mask_combine
img = casacore.images.image(mask_name)
pixels_mask = img.getdata()
imgcomb = casacore.images.image(mask_combine)
assert imgcomb.shape() == img.shape()
pixels_mask[np.where(imgcomb.getdata() == 1.)] = 1.
img.putdata(pixels_mask)
img.unlock()
imgcomb.unlock()
del img
del imgcomb
return mask_name
def find_sources_to_peel(image_name, threshpix=5, threshisl=3, atrous_do=False, catalog_name=None, rmsbox=(55,12)):
import lofar.bdsm as bdsm
# wavelets are required to fit gaussians
if atrous_do: stop_at = None
else: stop_at = 'isl'
# DO THE SOURCE DETECTION
img = bdsm.process_image(image_name, mean_map='zero', rms_box=rmsbox, \
thresh_pix=int(threshpix), thresh_isl=int(threshisl), atrous_do=atrous_do, atrous_jmax=3, \
adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(80,20), \
stop_at=stop_at, blank_limit=1e-5, quiet=True)
# SAVE THE CATALOG
img.write_catalog(outfile=catalog_name, format='fits', clobber=True)
def makecat(file):
print 'Making catalogue for', file
myout = file + '.catalog'
if not (os.path.exists(myout)):
img = bdsm.process_image(file,
thresh_pix=5,
detection_image='adaptive-stack.fits',
fix_to_beam=True,
rms_box=(55, 12),
adaptive_rms_box=True,
adaptive_thresh=150,
rms_box_bright=(80, 20),
mean_map='zero')
img.write_catalog(outfile=myout, clobber='True', format='ascii')
else:
print 'Catalog file exists'
def bdsm(lpar): ''' This runs PyBDSM on the image to make a mask from the Gaussian output from the source finder. ''' # Export the Image as Fits mirout, mirerr = lib.mirrun(task='fits', in_=lpars.image, op='xyout', out=lpars.image+'.fits') # Run PyBDSM on the Image img = bdsm.process_image(params.image+'.fits') img.export_image(outfile = params.image+'gaus_model.fits', im_type='gaus_model') img.write_catalog(outfile = params.image+'gaus_model.txt') # Use the Gaussian Image File as a Model mirout, mirerr = lib.mirrun(task='fits', in_=params.image+'gaus_model.fits', op='xyin', out=lpars.lsm) lpars.model = lpars.lsm tout = selfcal(lpars)
def makecat(file):
print 'Making catalogue for', file
myout = file + '.catalog.fits'
if not (os.path.exists(myout)):
img = bdsm.process_image(file,
thresh_pix=10,
thresh_isl=10,
detection_image='adaptive-stack-0.fits',
rms_map=True,
rms_box=(80, 20),
adaptive_rms_box=True,
adaptive_thresh=80,
rms_box_bright=(40, 10),
mean_map='zero')
img.write_catalog(outfile=myout,
clobber='True',
format='fits',
catalog_type='srl')
else:
print 'Catalog file exists'
def obsPreprocessSrcExtractionFunc(self):
#extract the source model with pybdsm
print ''
print 'extraction by pybdsm: bdsm.process_image %s,adaptive_rms_box=True,advanced_opts=True,detection_image=%s,thresh_isl=%s,thresh_pix=%s,rms_box=(%s,%s),blank_limit=1E-4,atrous_do=True'%('%sImage_substraction%s.restored.corr'%(self.preprocessImageDir,self.i),'%sImage_substraction%s.restored'%(self.preprocessImageDir,self.i),self.thresh_isl,self.thresh_pix,40,10)
print ''
#extract the source model with pybdsm
img = bdsm.process_image('%sImage_substraction%s.restored.corr'%(self.preprocessImageDir,self.i),adaptive_rms_box='True',advanced_opts='True',detection_image='%sImage_substraction%s.restored'%(self.preprocessImageDir,self.i),thresh_isl='%s'%(self.thresh_isl),thresh_pix='%s'%(self.thresh_pix),rms_box=(40,10),blank_limit=1E-4,atrous_do='True',ini_method='curvature')
#write bbs catalog
img.write_catalog(outfile="""%sSkymodel_substraction%s"""%(self.preprocessSkymodelDir,self.i),catalog_type='gaul',format='bbs',correct_proj='True')
#write ds9 catalog
img.write_catalog(outfile="""%sSkymodel_substraction%s.reg"""%(self.preprocessSkymodelDir,self.i),catalog_type='gaul',format='ds9',correct_proj='True')
#write fits catalog
img.write_catalog(outfile="""%sSkymodel_substraction%s.fits"""%(self.preprocessSkymodelDir,self.i),catalog_type='gaul',format='fits',correct_proj='True')
def find_sources_to_peel(image_name,
threshpix=5,
threshisl=3,
atrous_do=False,
catalog_name=None,
rmsbox=(55, 12)):
import lofar.bdsm as bdsm
# wavelets are required to fit gaussians
if atrous_do: stop_at = None
else: stop_at = 'isl'
# DO THE SOURCE DETECTION
img = bdsm.process_image(image_name, mean_map='zero', rms_box=rmsbox, \
thresh_pix=int(threshpix), thresh_isl=int(threshisl), atrous_do=atrous_do, atrous_jmax=3, \
adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(80,20), \
stop_at=stop_at, blank_limit=1e-5, quiet=True)
# SAVE THE CATALOG
img.write_catalog(outfile=catalog_name, format='fits', clobber=True)
def make_mask(image_name, threshpix=5, threshisl=3, atrous_do=False, mask_name=None, rmsbox=(55,12), mask_combine=None):
import sys, os
import numpy as np
import astropy.io.fits as pyfits
import casacore
import lofar.bdsm as bdsm
# wavelets are required to fit gaussians
if atrous_do: stop_at = None
else: stop_at = 'isl'
# DO THE SOURCE DETECTION
img = bdsm.process_image(image_name, mean_map='zero', rms_box=rmsbox, \
thresh_pix=int(threshpix), thresh_isl=int(threshisl), atrous_do=atrous_do, atrous_jmax=3, \
adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(23,10), \
stop_at=stop_at, blank_limit=1e-5, quiet=True)
# WRITE THE MASK FITS
if mask_name == None: mask_name = image_name+'.newmask'
print 'Making mask:', mask_name
img.export_image(img_type='island_mask', img_format='casa', outfile=mask_name, clobber=True)
del img
# do an pixel-by-pixel "OR" operation with a given mask
if mask_combine != None:
print "Combining with "+mask_combine
img = casacore.images.image(mask_name)
pixels_mask = img.getdata()
imgcomb = casacore.images.image(mask_combine)
assert imgcomb.shape() == img.shape()
pixels_mask[np.where(imgcomb.getdata() == 1.)] = 1.
img.putdata(pixels_mask)
img.unlock()
imgcomb.unlock()
del img
del imgcomb
return mask_name
def do_makecleanmask(image_name, threshpix, threshisl, atrousdo, ncores=8):
mask_name = image_name.split('.image')[0] + '.cleanmask'
logging.info('makecleanmask: Making mask: ' + mask_name)
if atrousdo:
threshisl = 4.0
logging.info(
'Changing island threshold to %.1f because atrous_do=True' %
threshisl)
os.system('rm -rf ' + mask_name)
# DO THE SOURCE DETECTION
#img = bdsm.process_image(image_name, mean_map='zero', rms_box=(70,10), thresh_pix=numpy.float(o.threshpix), \
# thresh_isl=numpy.float(o.threshisl), atrous_do=o.atrous_do,ini_method='curvature')
img = bdsm.process_image(image_name,
mean_map='zero',
rms_box=(80, 20),
thresh_pix=threshpix,
thresh_isl=threshisl,
atrous_do=atrousdo,
ini_method='curvature',
adaptive_rms_box=True,
adaptive_thresh=150,
rms_box_bright=(35, 7),
rms_map=True,
atrous_jmax=3,
ncores=ncores)
#img.show_fit()
# WRITE THE ISLAND MASK
img.export_image(img_type='island_mask',
img_format='casa',
outfile=mask_name,
clobber=True)
def do_makecleanmask_field_wsclean(image_name,
threshpix,
threshisl,
atrousdo,
ncores=8):
mask_name = image_name.split('-image')[0] + '.fitsmask'
logging.info('makecleanmask_field_wsclean: Making mask: ' + mask_name)
if atrousdo:
threshisl = 3.0
logging.info(
'Changing island threshold to %.1f because atrous_do=True' %
threshisl)
os.system('rm -rf ' + mask_name)
# DO THE SOURCE DETECTION
img = bdsm.process_image(image_name,
mean_map='zero',
rms_box=(300, 60),
thresh_pix=threshpix,
thresh_isl=threshisl,
atrous_do=atrousdo,
adaptive_rms_box=True,
adaptive_thresh=150,
rms_box_bright=(70, 10),
atrous_jmax=3,
ncores=ncores)
#img.show_fit()
# WRITE THE MASK FITS
img.export_image(img_type='island_mask',
img_format='fits',
outfile=mask_name)
def run(self, input_image, bdsm_parameter_run1_path,
bdsm_parameter_run2x_path, catalog_output_path, image_output_path,
sourcedb_target_path, environment, working_directory,
create_sourcdb_exec):
"""
:param input_image: image to look for sources in
:param bdsm_parameter_run1_path: parset with bdsm parameters for the
first run
:param bdsm_parameter_run2x_path: second ron bdsm parameters
:param catalog_output_path: Path to full list of sources found
:param image_output_path: Path to fits image with all sources
substracted
:param sourcedb_target_path: Path to store the sourcedb created from
containing all the found sources
:param environment: environment for runwithlog4cplus
:param working_directory: Working dir
:param create_sourcdb_exec: Path to create sourcedb executable
:rtype: self.outputs['source_db'] sourcedb_target_path
"""
#******************************************************************
# 0. Create the directories used in this recipe
create_directory(working_directory)
import lofar.bdsm as bdsm #@UnresolvedImport
self.logger.info("Starting imager_source_finding")
self.environment.update(environment)
# default frequency is None (read from image), save for later cycles.
# output of pybdsm forgets freq of source image
frequency = None
# Output of the for loop: n iterations and any source found
n_itter_sourcefind = None
sources_found = False
max_sourcefind_itter = 5 # TODO: maximum itter is a magic value
for idx in range(max_sourcefind_itter):
# ******************************************************************
# 1. Select correct input image
# The first iteration uses the input image, second and later use the
# output of the previous iteration. The 1+ iteration have a
# seperate parameter set.
if idx == 0:
input_image_local = input_image # input_image_cropped
image_output_path_local = image_output_path + "_0"
bdsm_parameter_local = parameterset(bdsm_parameter_run1_path)
else:
input_image_local = image_output_path + "_{0}".format(
str(idx - 1))
image_output_path_local = image_output_path + "_{0}".format(
str(idx))
bdsm_parameter_local = parameterset(bdsm_parameter_run2x_path)
# *****************************************************************
# 2. parse the parameters and convert to python if possible
# this is needed for pybdsm
bdsm_parameters = {}
for key in list(bdsm_parameter_local.keys()):
parameter_value = bdsm_parameter_local.getStringVector(key)[0]
try:
parameter_value = eval(parameter_value)
except:
pass #do nothing
bdsm_parameters[key] = parameter_value
# pybdsm needs its filename here, to derive the log location
bdsm_parameters["filename"] = input_image_local
# *****************************************************************
# 3. Start pybdsm
self.logger.debug(
"Starting sourcefinder bdsm on {0} using parameters:".format(
input_image_local))
self.logger.debug(repr(bdsm_parameters))
img = bdsm.process_image(bdsm_parameters, frequency=frequency)
# Always export the catalog
img.write_catalog(outfile=catalog_output_path +
"_{0}".format(str(idx)),
catalog_type='gaul',
clobber=True,
format="bbs",
force_output=True)
# If no more matching of sources with gausians is possible (nsrc==0)
# break the loop
if img.nsrc == 0:
n_itter_sourcefind = idx
break
# We have at least found a single source!
self.logger.debug("Number of source found: {0}".format(img.nsrc))
# *****************************************************************
# 4. export the image
self.logger.debug("Wrote list of sources to file at: {0})".format(
catalog_output_path))
img.export_image(outfile=image_output_path_local,
img_type='gaus_resid',
clobber=True,
img_format="fits")
self.logger.debug("Wrote fits image with substracted sources"
" at: {0})".format(image_output_path_local))
# Save the frequency from image header of the original input file,
# This information is not written by pybdsm to the exported image
frequency = img.frequency
# if not set the maximum number of itteration us performed
if n_itter_sourcefind == None:
n_itter_sourcefind = max_sourcefind_itter
# ********************************************************************
# 5. The produced catalogs now need to be combined into a single list
# Call with the number of loops and the path to the files, only combine
# if we found sources
self.logger.debug(
"Writing source list to file: {0}".format(catalog_output_path))
self._combine_source_lists(n_itter_sourcefind, catalog_output_path)
# *********************************************************************
# 6. Convert sourcelist to sourcedb
self._create_source_db(catalog_output_path, sourcedb_target_path,
working_directory, create_sourcdb_exec, False)
# Assign the outputs
self.outputs["catalog_output_path"] = catalog_output_path
self.outputs["source_db"] = sourcedb_target_path
return 0
def main(image_name,
mask_name,
atrous_do=False,
threshisl=0.0,
threshpix=0.0,
rmsbox=None,
rmsbox_bright=(35, 7),
iterate_threshold=False,
adaptive_rmsbox=False,
img_format='fits',
threshold_format='float',
trim_by=0.0,
vertices_file=None,
atrous_jmax=6,
pad_to_size=None,
skip_source_detection=False,
region_file=None,
nsig=1.0,
reference_ra_deg=None,
reference_dec_deg=None,
cellsize_deg=0.000417,
use_adaptive_threshold=False,
adaptive_thresh=150.0):
"""
Make a clean mask and return clean threshold
Parameters
----------
image_name : str
Filename of input image from which mask will be made. If the image does
not exist, a template image with center at (reference_ra_deg,
reference_dec_deg) will be made internally
mask_name : str
Filename of output mask image
atrous_do : bool, optional
Use wavelet module of PyBDSM?
threshisl : float, optional
Value of thresh_isl PyBDSM parameter
threshpix : float, optional
Value of thresh_pix PyBDSM parameter
rmsbox : tuple of floats, optional
Value of rms_box PyBDSM parameter
rmsbox_bright : tuple of floats, optional
Value of rms_box_bright PyBDSM parameter
iterate_threshold : bool, optional
If True, threshold will be lower in 20% steps until
at least one island is found
adaptive_rmsbox : tuple of floats, optional
Value of adaptive_rms_box PyBDSM parameter
img_format : str, optional
Format of output mask image (one of 'fits' or 'casa')
threshold_format : str, optional
Format of output threshold (one of 'float' or 'str_with_units')
trim_by : float, optional
Fraction by which the perimeter of the output mask will be
trimmed (zeroed)
vertices_file : str, optional
Filename of file with vertices (must be a pickle file containing
a dictionary with the vertices in the 'vertices' entry)
atrous_jmax : int, optional
Value of atrous_jmax PyBDSM parameter
pad_to_size : int, optional
Pad output mask image to a size of pad_to_size x pad_to_size
skip_source_detection : bool, optional
If True, source detection is not run on the input image
region_file : str, optional
Filename of region file in CASA format. If given, no mask image
is made (the region file is used as the clean mask)
nsig : float, optional
Number of sigma of returned threshold value
reference_ra_deg : float, optional
RA for center of output mask image
reference_dec_deg : float, optional
Dec for center of output mask image
cellsize_deg : float, optional
Size of a pixel in degrees
use_adaptive_threshold : bool, optional
If True, use an adaptive threshold estimated from the negative values in
the image
adaptive_thresh : float, optional
If adaptive_rmsbox is True, this value sets the threshold above
which a source will use the small rms box
Returns
-------
result : dict
Dict with nsig-sigma rms threshold
"""
if rmsbox is not None and type(rmsbox) is str:
rmsbox = eval(rmsbox)
if type(rmsbox_bright) is str:
rmsbox_bright = eval(rmsbox_bright)
if pad_to_size is not None and type(pad_to_size) is str:
pad_to_size = int(pad_to_size)
if type(atrous_do) is str:
if atrous_do.lower() == 'true':
atrous_do = True
threshisl = 4.0 # override user setting to ensure proper source fitting
else:
atrous_do = False
if type(iterate_threshold) is str:
if iterate_threshold.lower() == 'true':
iterate_threshold = True
else:
iterate_threshold = False
if type(adaptive_rmsbox) is str:
if adaptive_rmsbox.lower() == 'true':
adaptive_rmsbox = True
else:
adaptive_rmsbox = False
if type(skip_source_detection) is str:
if skip_source_detection.lower() == 'true':
skip_source_detection = True
else:
skip_source_detection = False
if type(use_adaptive_threshold) is str:
if use_adaptive_threshold.lower() == 'true':
use_adaptive_threshold = True
else:
use_adaptive_threshold = False
if reference_ra_deg is not None and reference_dec_deg is not None:
reference_ra_deg = float(reference_ra_deg)
reference_dec_deg = float(reference_dec_deg)
if not os.path.exists(image_name):
print('Input image not found. Making empty image...')
if not skip_source_detection:
print('ERROR: Source detection cannot be done on an empty image')
sys.exit(1)
if reference_ra_deg is not None and reference_dec_deg is not None:
image_name = mask_name + '.tmp'
make_template_image(image_name,
reference_ra_deg,
reference_dec_deg,
cellsize_deg=float(cellsize_deg))
else:
print(
'ERROR: if image not found, a refernce position must be given')
sys.exit(1)
trim_by = float(trim_by)
atrous_jmax = int(atrous_jmax)
threshpix = float(threshpix)
threshisl = float(threshisl)
nsig = float(nsig)
adaptive_thresh = float(adaptive_thresh)
threshold = 0.0
if not skip_source_detection:
if vertices_file is not None:
# Modify the input image to blank the regions outside of the polygon
temp_img = pim.image(image_name)
image_name += '.blanked'
temp_img.saveas(image_name, overwrite=True)
input_img = pim.image(image_name)
data = input_img.getdata()
vertices = read_vertices(vertices_file)
RAverts = vertices[0]
Decverts = vertices[1]
xvert = []
yvert = []
for RAvert, Decvert in zip(RAverts, Decverts):
pixels = input_img.topixel(
[1, 1, Decvert * np.pi / 180.0, RAvert * np.pi / 180.0])
xvert.append(pixels[2]) # x -> Dec
yvert.append(pixels[3]) # y -> RA
poly = Polygon(xvert, yvert)
# Find masked regions
masked_ind = np.where(data[0, 0])
# Find distance to nearest poly edge and set to NaN those that
# are outside the facet (dist < 0)
dist = poly.is_inside(masked_ind[0], masked_ind[1])
outside_ind = np.where(dist < 0.0)
if len(outside_ind[0]) > 0:
data[0, 0, masked_ind[0][outside_ind],
masked_ind[1][outside_ind]] = np.nan
# Save changes
input_img.putdata(data)
if use_adaptive_threshold:
# Get an estimate of the rms
img = bdsm.process_image(image_name,
mean_map='zero',
rms_box=rmsbox,
thresh_pix=threshpix,
thresh_isl=threshisl,
atrous_do=atrous_do,
ini_method='curvature',
thresh='hard',
adaptive_rms_box=adaptive_rmsbox,
adaptive_thresh=adaptive_thresh,
rms_box_bright=rmsbox_bright,
rms_map=True,
quiet=True,
atrous_jmax=atrous_jmax,
stop_at='isl')
# Find min and max pixels
max_neg_val = abs(np.min(img.ch0_arr))
max_neg_pos = np.where(img.ch0_arr == np.min(img.ch0_arr))
max_pos_val = abs(np.max(img.ch0_arr))
max_pos_pos = np.where(img.ch0_arr == np.max(img.ch0_arr))
# Estimate new thresh_isl from min pixel value's sigma, but don't let
# it get higher than 1/2 of the peak's sigma
threshisl_neg = 2.0 * max_neg_val / img.rms_arr[max_neg_pos][0]
max_sigma = max_pos_val / img.rms_arr[max_pos_pos][0]
if threshisl_neg > max_sigma / 2.0:
threshisl_neg = max_sigma / 2.0
# Use the new threshold only if it is larger than the user-specified one
if threshisl_neg > threshisl:
threshisl = threshisl_neg
if iterate_threshold:
# Start with given threshold and lower it until we get at least one island
nisl = 0
while nisl == 0:
img = bdsm.process_image(image_name,
mean_map='zero',
rms_box=rmsbox,
thresh_pix=threshpix,
thresh_isl=threshisl,
atrous_do=atrous_do,
ini_method='curvature',
thresh='hard',
adaptive_rms_box=adaptive_rmsbox,
adaptive_thresh=adaptive_thresh,
rms_box_bright=rmsbox_bright,
rms_map=True,
quiet=True,
atrous_jmax=atrous_jmax)
nisl = img.nisl
threshpix /= 1.2
threshisl /= 1.2
if threshpix < 5.0:
break
else:
img = bdsm.process_image(image_name,
mean_map='zero',
rms_box=rmsbox,
thresh_pix=threshpix,
thresh_isl=threshisl,
atrous_do=atrous_do,
ini_method='curvature',
thresh='hard',
adaptive_rms_box=adaptive_rmsbox,
adaptive_thresh=adaptive_thresh,
rms_box_bright=rmsbox_bright,
rms_map=True,
quiet=True,
atrous_jmax=atrous_jmax)
if img.nisl == 0:
if region_file is None or region_file == '[]':
print('No islands found. Clean mask cannot be made.')
sys.exit(1)
else:
# Continue on and use user-supplied region file
skip_source_detection = True
threshold = nsig * img.clipped_rms
# Check if there are large islands preset (indicating that multi-scale
# clean is needed)
has_large_isl = False
for isl in img.islands:
if isl.size_active > 100:
# Assuming normal sampling, a size of 100 pixels would imply
# a source of ~ 10 beams
has_large_isl = True
if (region_file is not None and region_file != '[]'
and skip_source_detection):
# Copy region file and return if source detection was not done
os.system('cp {0} {1}'.format(region_file.strip('[]"'), mask_name))
if threshold_format == 'float':
return {'threshold_5sig': threshold}
elif threshold_format == 'str_with_units':
# This is done to get around the need for quotes around strings in casapy scripts
# 'casastr/' is removed by the generic pipeline
return {'threshold_5sig': 'casastr/{0}Jy'.format(threshold)}
elif not skip_source_detection:
img.export_image(img_type='island_mask',
mask_dilation=0,
outfile=mask_name,
img_format=img_format,
clobber=True)
if (vertices_file is not None or trim_by > 0 or pad_to_size is not None
or (region_file is not None and region_file != '[]')
or skip_source_detection):
# Alter the mask in various ways
if skip_source_detection:
# Read the image
mask_im = pim.image(image_name)
else:
# Read the PyBDSM mask
mask_im = pim.image(mask_name)
data = mask_im.getdata()
coordsys = mask_im.coordinates()
if reference_ra_deg is not None and reference_dec_deg is not None:
values = coordsys.get_referencevalue()
values[2][0] = reference_dec_deg / 180.0 * np.pi
values[2][1] = reference_ra_deg / 180.0 * np.pi
coordsys.set_referencevalue(values)
imshape = mask_im.shape()
del (mask_im)
if pad_to_size is not None:
imsize = pad_to_size
coordsys['direction'].set_referencepixel([imsize / 2, imsize / 2])
pixmin = (imsize - imshape[2]) / 2
if pixmin < 0:
print("The padded size must be larger than the original size.")
sys.exit(1)
pixmax = pixmin + imshape[2]
data_pad = np.zeros((1, 1, imsize, imsize), dtype=np.float32)
data_pad[0, 0, pixmin:pixmax, pixmin:pixmax] = data[0, 0]
new_mask = pim.image('',
shape=(1, 1, imsize, imsize),
coordsys=coordsys)
new_mask.putdata(data_pad)
else:
new_mask = pim.image('', shape=imshape, coordsys=coordsys)
new_mask.putdata(data)
data = new_mask.getdata()
if skip_source_detection:
# Mask all pixels
data[:] = 1
if vertices_file is not None:
# Modify the clean mask to exclude regions outside of the polygon
vertices = read_vertices(vertices_file)
RAverts = vertices[0]
Decverts = vertices[1]
xvert = []
yvert = []
for RAvert, Decvert in zip(RAverts, Decverts):
try:
pixels = new_mask.topixel([
0, 1, Decvert * np.pi / 180.0, RAvert * np.pi / 180.0
])
except:
pixels = new_mask.topixel([
1, 1, Decvert * np.pi / 180.0, RAvert * np.pi / 180.0
])
xvert.append(pixels[2]) # x -> Dec
yvert.append(pixels[3]) # y -> RA
poly = Polygon(xvert, yvert)
# Find masked regions
masked_ind = np.where(data[0, 0])
# Find distance to nearest poly edge and unmask those that
# are outside the facet (dist < 0)
dist = poly.is_inside(masked_ind[0], masked_ind[1])
outside_ind = np.where(dist < 0.0)
if len(outside_ind[0]) > 0:
data[0, 0, masked_ind[0][outside_ind],
masked_ind[1][outside_ind]] = 0
if trim_by > 0.0:
sh = np.shape(data)
margin = int(sh[2] * trim_by / 2.0)
data[0, 0, 0:sh[2], 0:margin] = 0
data[0, 0, 0:margin, 0:sh[3]] = 0
data[0, 0, 0:sh[2], sh[3] - margin:sh[3]] = 0
data[0, 0, sh[2] - margin:sh[2], 0:sh[3]] = 0
if region_file is not None and region_file != '[]':
# Merge the CASA regions with the mask
casa_polys = read_casa_polys(region_file.strip('[]"'), new_mask)
for poly in casa_polys:
# Find unmasked regions
unmasked_ind = np.where(data[0, 0] == 0)
# Find distance to nearest poly edge and mask those that
# are inside the casa region (dist > 0)
dist = poly.is_inside(unmasked_ind[0], unmasked_ind[1])
inside_ind = np.where(dist > 0.0)
if len(inside_ind[0]) > 0:
data[0, 0, unmasked_ind[0][inside_ind],
unmasked_ind[1][inside_ind]] = 1
# Save changes
new_mask.putdata(data)
if img_format == 'fits':
new_mask.tofits(mask_name, overwrite=True)
elif img_format == 'casa':
new_mask.saveas(mask_name, overwrite=True)
else:
print(
'Output image format "{}" not understood.'.format(img_format))
sys.exit(1)
if not skip_source_detection:
if threshold_format == 'float':
return {
'threshold_5sig': nsig * img.clipped_rms,
'multiscale': has_large_isl
}
elif threshold_format == 'str_with_units':
# This is done to get around the need for quotes around strings in casapy scripts
# 'casastr/' is removed by the generic pipeline
return {
'threshold_5sig':
'casastr/{0}Jy'.format(nsig * img.clipped_rms),
'multiscale': has_large_isl
}
else:
return {'threshold_5sig': '0.0'}
import lofar.bdsm as bdsm
import sys
# Process the image
img = bdsm.process_image('tbdsm_process_image.in')
# List of operations that must have been done on `img`.
operations = [
'readimage', 'collapse', 'preprocess', 'rmsimage', 'threshold', 'islands',
'gausfit', 'gaul2srl', 'make_residimage', 'wavelet_atrous', 'shapelets',
'spectralindex', 'polarisation', 'psf_vary', 'cleanup'
]
# Return exit status 0 if everything went fine, otherwise return 1.
if img and all(oper in img.completed_Ops for oper in operations):
sys.exit(0)
else:
sys.exit(1)
def selfCalRunFuncSrcExtraction(self):
if self.i < self.nbCycle:
#extract the source model with pybdsm
print ''
print 'extraction by pybdsm: bdsm.process_image %s,adaptive_rms_box=True,advanced_opts=True,detection_image=%s,thresh_isl=6,thresh_pix=8,rms_box=(%s,%s),blank_limit=1E-4,atrous_do=True' % (
"""%sImage_%sarcsec_Iter%s.restored.corr""" %
(self.ImagePathDir, self.pixsize[self.i], self.i),
"""%sImage_%sarcsec_Iter%s.restored""" %
(self.ImagePathDir, self.pixsize[self.i], self.i),
self.RMS_BOX[0], self.RMS_BOX[1])
print ''
#extract the source model with pybdsm
img = bdsm.process_image(
"""%sImage_%sarcsec_Iter%s.restored.corr""" %
(self.ImagePathDir, self.pixsize[self.i], self.i),
adaptive_rms_box='True',
advanced_opts='True',
detection_image="""%sImage_%sarcsec_Iter%s.restored""" %
(self.ImagePathDir, self.pixsize[self.i], self.i),
thresh_isl=6,
thresh_pix=8,
rms_box=(self.RMS_BOX[0], self.RMS_BOX[1]),
blank_limit=1E-4,
atrous_do='True')
#write bbs catalog
img.write_catalog(outfile="""%sSkymodel_Iter%s""" %
(self.SkymodelPath, self.i + 1),
catalog_type='gaul',
format='bbs',
correct_proj='True')
#write ds9 catalog
img.write_catalog(outfile="""%sSkymodel_Iter%s.reg""" %
(self.SkymodelPath, self.i + 1),
catalog_type='gaul',
format='ds9',
correct_proj='True')
#write fits catalog
img.write_catalog(outfile="""%sSkymodel_Iter%s.fits""" %
(self.SkymodelPath, self.i + 1),
catalog_type='gaul',
format='fits',
correct_proj='True')
if self.i == self.nbCycle:
#extract the source model with pybdsm
print ''
print 'extraction by pybdsm: bdsm.process_image %s,adaptive_rms_box=True,advanced_opts=True,detection_image=%s,thresh_isl=6,thresh_pix=8,rms_box=(%s,%s),blank_limit=1E-4,atrous_do=True' % (
"""%sFinal_Image_%sarcsec_Iter%s.restored.corr""" %
(self.ImagePathDir, self.pixsize[self.i - 1], self.i),
"""%sFinal_Image_%sarcsec_Iter%s.restored""" %
(self.ImagePathDir, self.pixsize[self.i - 1], self.i),
self.RMS_BOX[0], self.RMS_BOX[1])
print ''
#extract the source model with pybdsm
img = bdsm.process_image(
"""%sFinal_Image_%sarcsec_Iter%s.restored.corr""" %
(self.ImagePathDir, self.pixsize[self.i - 1], self.i),
adaptive_rms_box='True',
advanced_opts='True',
detection_image="""%sFinal_Image_%sarcsec_Iter%s.restored""" %
(self.ImagePathDir, self.pixsize[self.i - 1], self.i),
thresh_isl=6,
thresh_pix=8,
rms_box=(self.RMS_BOX[0], self.RMS_BOX[1]),
blank_limit=1E-4,
atrous_do='True')
#write bbs catalog
img.write_catalog(outfile="""%sFinal_Skymodel_Iter%s""" %
(self.SkymodelPath, self.i + 1),
catalog_type='gaul',
format='bbs',
correct_proj='True')
#write ds9 catalog
img.write_catalog(outfile="""%sFinal_Skymodel_Iter%s.reg""" %
(self.SkymodelPath, self.i + 1),
catalog_type='gaul',
format='ds9',
correct_proj='True')
#write fits catalog
img.write_catalog(outfile="""%sFinal_Skymodel_Iter%s.fits""" %
(self.SkymodelPath, self.i + 1),
catalog_type='gaul',
format='fits',
correct_proj='True')
os.system('mv old_run_%s/startingfiles .' % starttime)
else:
os.system('mkdir %s' % workingdir)
os.system('cp -r %s %s/%s' %
(orig_lofar_msfile, workingdir, new_lofar_msfile))
os.system('cp -r %s %s/%s' %
(orig_lofar_image, workingdir, new_lofar_image))
os.chdir(workingdir)
os.system('mkdir startingfiles')
os.system('cp -r ./* startingfiles/')
# Extract the N brightest sources from the initial image.
img = bdsm.process_image('startingfiles/%s' % new_lofar_image,
adaptive_rms_box='True',
advanced_opts='True',
detection_image="startingfiles/%s" % new_lofar_image,
thresh_isl=6,
thresh_pix=8,
blank_limit=1E-4,
atrous_do='True')
img.write_catalog(outfile="%s_sources.fits" % new_lofar_image,
catalog_type='gaul',
format='fits',
correct_proj='True')
img.export_image(outfile="%s_pydbsm_mask" % new_lofar_image,
img_type='island_mask',
img_format='casa')
f = pyfits.open('%s_sources.fits' % new_lofar_image)
fluxes = f[1].data['PEAK_FLUX'].copy()
f.close()
fluxes.sort()
fluxes = sorted(fluxes, reverse=True)
workingimage= 'IMAGE_%s_%s'%(round(resolution,2),i)
if i == 0:
iterations,threshold = 300000,'%smJy'%(np.max([infodict['Est Noise']*10.0,brightestsource*1000.0/400.0]))
prevthreshold = np.max([infodict['Est Noise']*10.0,brightestsource*1000.0/400.0])
else:
iterations,threshold = 300000,'%smJy'%(np.max([rms*8.0*1000,brightestsource*1000.0/400.0])) # Do not clean so deeply without the clean boxes
prevthreshold = np.max([rms*8.0*1000,brightestsource*1000.0/400.0])
flag_measurementset(new_lofar_imagemsfile,new_lofar_imagemsfile,'CORRECTED_DATA','ndppp_flag_%s.log'%i)
aw_imagedata('%s'%new_lofar_imagemsfile,'aw_imager_%s.inp'%i,'%s'%workingimage,iterations,threshold,cellsize,uvmin,uvmax,npix,'mfclark','',robust)
os.system("echo 'Iter%s initial imaging time %s \n' >> %s"%(i,time.time()-previoustime,loggingfilename))
previoustime = time.time()
img = bdsm.process_image('%s.restored'%workingimage,advanced_opts='True',detection_image='%s.restored'%workingimage,thresh_isl=3,thresh_pix=5,blank_limit=1E-4,adaptive_rms_box='True',adaptive_thresh=200)#,adaptive_rms_box='True',atrous_do='True')
img.export_image(outfile="mask_%s"%workingimage,img_type='island_mask',img_format='casa')
img.export_image(outfile="rms_%s"%workingimage,img_type='rms',img_format='casa')
os.system('image2fits in=%s.restored out=%s.restored.fits'%(workingimage,workingimage))
f = pyfits.open('%s.restored.fits'%workingimage)
noisearray = f[0].data.flatten()
maxpixel = np.max(noisearray)
noisearray = np.random.permutation(noisearray)[:10000]
noisepix = np.array(filter(lambda x: abs(x) > 10E-8,noisearray))
noisepix = np.array(filter(lambda x: abs(x)<infodict['Est Noise']*50.0/1000.0,noisepix))
rms = fit_gaussian_histogram(noisepix,'n')
print 'rms %s, maxpixel %s'%(rms,maxpixel)
f.close()
minthreshold = rms
image = pim.image("rms_%s"%workingimage)
#!/usr/bin/python import lofar.bdsm as bdsm import sys file = sys.argv[1] img = bdsm.process_image(file, advanced_opts=True, blank_zeros=True, interactive=True, thresh_pix=6., thresh_isl=4.5) img.write_catalog(format='fits',catalog_type='srl', clobber=True) # for bdsm2cat.py img.write_catalog(format='bbs', bbs_patches=None,catalog_type='gaul', clobber=True) # for sagecal to trnsform img.write_catalog(format='ds9',catalog_type='gaul', clobber=True) # for ds9 img.export_image(outfile=file+'_gaus_resid.fits', img_type='gaus_resid',clobber=True) img.export_image(outfile=file+'_gaus_model.fits', img_type='gaus_model',clobber=True) img.export_image(outfile=file+'_rms.fits', img_type='rms',clobber=True) img.export_image(outfile=file+'_mean.fits', img_type='mean',clobber=True) #~/scripts/bdsm2cat.py --snr 1 --sr 30 --phase_center "187.705833, 12.391111" awimager-145.pybdsm.srl.fits
updateHistory(log,sb,proc,'f')
# Run PyBDSM and output maps,stats etc.
# This could be run directly on the MS.restored.corr in fact
if 'x' in ops and 'f' in sbh:
if not os.path.exists('%s.restored.corr.fits'%imgout):
print '%s.restored.corr.fits is missing!'%imgout
continue
do_rms_map=True
kappa_clip=3.0
print 'Launching PyBDSM...'
ncores=None # i.e. auto
clobber=True
psf_vary_do=False
img=bdsm.process_image('%s.restored.corr.fits'%imgout,\
rms_map=do_rms_map,kappa_clip=kappa_clip,\
psf_vary_do=psf_vary_do,ncores=ncores)
img_format='fits'
img.export_image(img_type='rms',img_format=img_format,clobber=clobber)
img.export_image(img_type='ch0',img_format=img_format,clobber=clobber)
try:
img.export_image(img_type='psf_pa',img_format=img_format,clobber=clobber)
except AttributeError:
print 'Could not write psf_pa image for some reason..'
catalog_type='srl'
img.write_catalog(format='ascii',clobber=clobber,catalog_type=catalog_type)
img.write_catalog(format='fits',clobber=clobber,catalog_type=catalog_type)
catalog_type='gaul'
def pybdsm_search (image="${imager.RESTORED_IMAGE}",output="$PYBDSM_OUTPUT",pol='$PYBDSM_POLARIZED',
select=None,center=None,
threshold=None,pbexp=None,**kw):
"""Runs pybdsm on the specified 'image', converts the results into a Tigger model and writes it to 'output'.
Use 'threshold' to specify a non-default threshold (thresh_isl and thresh_pix).
Use 'pol' to force non-default polarized mode.
Use 'pbexp' to supply a primary beam expression (passed to tigger-convert), in which case the output model will contain
intrinsic fluxes.
Use 'select' to apply a selection string on the new model (e.g. "I.gt.0.001")
Use 'center' to set the centre of the model (useful for e.g. selecting on radius), use a string e.g. Xdeg,Ydeg
"""
image,output,pol,center = interpolate_locals("image output pol center");
makedir(v.DESTDIR);
# setup parameters
gaul = II("${output:BASEPATH}.gaul");
# info("PyBDSM filenames are $output $gaul");
# start with default PYBDSM options
opts = PYBDSM_OPTIONS.copy();
opts.update(kw);
# override with explicit arguments
if threshold:
opts['thresh_pix'] = threshold;
if pol is not None:
opts['polarisation_do'] = is_true(pol);
pol = opts.get('polarisation_do',False);
opts['quiet'] = True;
# run pybdsm
info("running PyBDSM process_image($image,%s)"%",".join(sorted([ "%s=%s"%x for x in opts.iteritems() ])));
from lofar import bdsm
img = bdsm.process_image(image,**opts);
info("writing PyBDSM gaul catalog");
img.write_catalog(outfile=gaul,format='ascii',catalog_type='gaul',clobber=True);
# add log to output
logfile = II("${output:BASEPATH}.pybdsm.log");
if exists(logfile):
info("PyBDSM log output follows:");
for line in file(logfile):
print " ",line;
else:
warn("PyBDSM log $logfile not found");
# set clustering parameter from beam size
cluster = CLUSTER_DIST;
if not cluster:
hdr = pyfits.open(image)[0].header;
# BMAJ/BMIN is in degrees -- convert to seconds, or fall back to 60" if not set
cluster = 1800*(hdr.get('BMAJ',0)+hdr.get('BMIN',0))*CLUSTER_DIST_BEAMS or 60;
# convert catalog
if pbexp:
args = [ "--primary-beam",pbexp,"--app-to-int" ]
else:
args = []
if select:
args += [ "--select",select ];
if center:
args += [ "--center",center ]
## leaving this for now but it seems overly complicated. Better to add an option
## to tigger-convert to eliminate sources with NaN positions
verifyGaulModel(gaul)
tigger_convert(gaul,output,"-t","Gaul",
"-f","--rename",
"--cluster-dist",cluster,
"--min-extent",MIN_EXTENT,
split_args=False,
*args);
def pybdsm_search (image="${imager.RESTORED_IMAGE}",output="$PYBDSM_OUTPUT",pol='$PYBDSM_POLARIZED',
select=None,
threshold=None,pbexp=None,**kw):
"""Runs pybdsm on the specified 'image', converts the results into a Tigger model and writes it to 'output'.
Use 'threshold' to specify a non-default threshold (thresh_isl and thresh_pix).
Use 'pol' to force non-default polarized mode.
Use 'pbexp' to supply a primary beam expression (passed to tigger-convert), in which case the output model will contain
intrinsic fluxes.
Use 'select' to apply a selection string on the new model (e.g. "I.gt.0.001")
"""
image,output,pol = interpolate_locals("image output pol");
makedir(v.DESTDIR);
# setup parameters
gaul = II("${output:FILE}.gaul");
# start with default PYBDSM options
opts = PYBDSM_OPTIONS.copy();
opts.update(kw);
# override with explicit arguments
if threshold:
opts['thresh_pix'] = threshold;
if pol is not None:
opts['polarisation_do'] = is_true(pol);
pol = opts.get('polarisation_do',False);
opts['quiet'] = True;
# run pybdsm
info("running PyBDSM process_image($image,%s)"%",".join(sorted([ "%s=%s"%x for x in opts.iteritems() ])));
from lofar import bdsm
img = bdsm.process_image(image,**kw);
info("writing PyBDSM gaul catalog");
img.write_catalog(outfile=gaul,format='ascii',catalog_type='gaul',clobber=True);
# add log to output
logfile = II("${output:FILE}.pybdsm.log");
if exists(logfile):
info("PyBDSM log output follows:");
for line in file(logfile):
print " ",line;
else:
warn("PyBDSM log $logfile not found");
# set clustering parameter from beam size
cluster = CLUSTER_DIST;
if not cluster:
hdr = pyfits.open(image)[0].header;
# BMAJ/BMIN is in degrees -- convert to seconds, or fall back to 60" if not set
cluster = 1800*(hdr.get('BMAJ',0)+hdr.get('BMIN',0))*CLUSTER_DIST_BEAMS or 60;
# convert catalog
if pbexp:
args = [ "--primary-beam",pbexp,"--app-to-int" ]
else:
args = []
if select:
args += [ "--select",select ];
verifyGaulModel(gaul)
tigger_convert(gaul,output,"-t","ASCII","--format",
"name Isl_id Source_id Wave_id ra_d E_RA dec_d E_DEC i E_Total_flux Peak_flux E_Peak_flux Xposn E_Xposn Yposn E_Yposn Maj E_Maj Min E_Min PA E_PA emaj_d E_DC_Maj emin_d E_DC_Min pa_d E_DC_PA Isl_Total_flux E_Isl_Total_flux Isl_rms Isl_mean Resid_Isl_rms Resid_Isl_mean S_Code"
+
("q E_Total_Q u E_Total_U v E_Total_V Linear_Pol_frac Elow_Linear_Pol_frac Ehigh_Linear_Pol_frac "+
"Circ_Pol_Frac Elow_Circ_Pol_Frac Ehigh_Circ_Pol_Frac Total_Pol_Frac Elow_Total_Pol_Frac Ehigh_Total_Pol_Frac Linear_Pol_Ang E_Linear_Pol_Ang"
if pol else ""),
"-f","--rename",
"--cluster-dist",cluster,
"--min-extent",MIN_EXTENT,
split_args=False,
*args);
def selfCalRunFuncSrcExtraction(self,stepProcess):
################################################################
# Using a mask for cleaning
################################################################
if self.mask == 'yes':
# 0: Initial: needed for generate the mask
if stepProcess == 0:
if self.i < self.nbCycle:
#extract the source model with pybdsm
print ''
print 'extraction by pybdsm: bdsm.process_image %s,adaptive_rms_box=True,advanced_opts=True,detection_image=%s,thresh_isl=%s,thresh_pix=%s,rms_box=(%s,%s),rms_box_bright=(%s,%s),adaptive_thresh=30,blank_limit=1E-4,atrous_do=True,ini_method=curvature_do=True)#,psf_vary_do=True,psf_stype_only=False,psf_snrcut=5,psf_snrcutstack=5,psf_snrtop=0.3'%("""%sTemporary_Image_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i],self.i),"""%sTemporary_Image_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i],self.i),self.thresh_isl,self.thresh_pix,self.RMS_BOX[0],self.RMS_BOX[1],self.RMS_BOX_Bright[0],self.RMS_BOX_Bright[1])
print ''
#extract the source model with pybdsm
img = bdsm.process_image("""%sTemporary_Image_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i],self.i),adaptive_rms_box='True',advanced_opts='True',detection_image="""%sTemporary_Image_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i],self.i),thresh_isl='%s'%(self.thresh_isl),thresh_pix='%s'%(self.thresh_pix),rms_box=(self.RMS_BOX[0],self.RMS_BOX[1]),rms_box_bright=(self.RMS_BOX_Bright[0],self.RMS_BOX_Bright[1]),adaptive_thresh=30,blank_limit=1E-4,atrous_do='True',ini_method='curvature')#,psf_vary_do='True',psf_stype_only='False',psf_snrcut=5,psf_snrcutstack=5,psf_snrtop=0.3)
#Skymodel
#write bbs catalog
img.write_catalog(outfile="""%sTemporary_Pybdsm_Skymodel_Iter%s"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='bbs',correct_proj='True',clobber='True')
#write ds9 catalog
img.write_catalog(outfile="""%sTemporary_Pybdsm_Skymodel_Iter%s.reg"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='ds9',correct_proj='True',clobber='True')
#Catalog
#write ds9 catalog
img.write_catalog(outfile="""%sTemporary_Catalog_Iter%s.reg"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='ds9',correct_proj='True',clobber='True')
#write fits catalog
img.write_catalog(outfile="""%sTemporary_Catalog_Iter%s.fits"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='fits',correct_proj='True',clobber='True')
#write bbs catalog
#img.write_catalog(outfile="""%sTemporary_Catalog_Iter%s"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='bbs',correct_proj='True',clobber='True')
# Extract the mask in fits format
img.export_image(outfile="""%sMask_Iter%s.casa"""%(self.SkymodelPath,self.i+1),img_format='casa',img_type='island_mask',mask_dilation=self.maskDilation)
if self.i == self.nbCycle:
#extract the source model with pybdsm
print ''
print 'extraction by pybdsm: bdsm.process_image %s,adaptive_rms_box=True,advanced_opts=True,detection_image=%s,thresh_isl=%s,thresh_pix=%s,rms_box=(%s,%s),rms_box_bright=(%s,%s),adaptive_thresh=30,blank_limit=1E-4,atrous_do=True,ini_method=curvature)#,psf_vary_do=True,psf_stype_only=False,psf_snrcut=5,psf_snrcutstack=5,psf_snrtop=0.3'%("""%sTemporary_Final_Image_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),"""%sTemporary_Final_Image_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),self.thresh_isl,self.thresh_pix,self.RMS_BOX[0],self.RMS_BOX[1],self.RMS_BOX_Bright[0],self.RMS_BOX_Bright[1])
print ''
#extract the source model with pybdsm
img = bdsm.process_image("""%sTemporary_Final_Image_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),adaptive_rms_box='True',advanced_opts='True',detection_image="""%sTemporary_Final_Image_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),thresh_isl='%s'%(self.thresh_isl),thresh_pix='%s'%(self.thresh_pix),rms_box=(self.RMS_BOX[0],self.RMS_BOX[1]),rms_box_bright=(self.RMS_BOX_Bright[0],self.RMS_BOX_Bright[1]),adaptive_thresh=30,blank_limit=1E-4,atrous_do='True',ini_method='curvature')#,psf_vary_do='True',psf_stype_only='False',psf_snrcut=5,psf_snrcutstack=5,psf_snrtop=0.3)
#Skymodel
#write bbs Skymodel
img.write_catalog(outfile="""%sTemporary_Final_Pybdsm_Skymodel_Iter%s"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='bbs',correct_proj='True',clobber='True')
#write ds9 Skymodel
img.write_catalog(outfile="""%sTemporary_Final_Pybdsm_Skymodel_Iter%s.reg"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='ds9',correct_proj='True',clobber='True')
#Catalog
#write ds9 catalog
img.write_catalog(outfile="""%sTemporary_Final_Catalog_Iter%s.reg"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='ds9',correct_proj='True',clobber='True')
#write fits catalog
img.write_catalog(outfile="""%sTemporary_Final_Catalog_Iter%s.fits"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='fits',correct_proj='True',clobber='True')
#write ds9 catalog
#img.write_catalog(outfile="""%sTemporary_Final_Catalog_Iter%s"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='bbs',correct_proj='True',clobber='True')
# Extract the mask in fits format
img.export_image(outfile="""%sFinal_Mask_Iter%s.casa"""%(self.SkymodelPath,self.i+1),img_format='casa',img_type='island_mask',mask_dilation=self.maskDilation)
# 1: cleaning with mask
if stepProcess == 1:
if self.i < self.nbCycle:
#extract the source model with pybdsm
print ''
print 'extraction by pybdsm: bdsm.process_image %s,adaptive_rms_box=True,advanced_opts=True,detection_image=%s,thresh_isl=%s,thresh_pix=%s,rms_box=(%s,%s),rms_box_bright=(%s,%s),adaptive_thresh=30,blank_limit=1E-4,atrous_do=True,ini_method=curvature)#,psf_vary_do=True,psf_stype_only=False,psf_snrcut=5,psf_snrcutstack=5,psf_snrtop=0.3'%("""%sImage_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i],self.i),"""%sImage_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i],self.i),self.thresh_isl,self.thresh_pix,self.RMS_BOX[0],self.RMS_BOX[1],self.RMS_BOX_Bright[0],self.RMS_BOX_Bright[1])
print ''
#extract the source model with pybdsm
img = bdsm.process_image("""%sImage_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i],self.i),adaptive_rms_box='True',advanced_opts='True',detection_image="""%sImage_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i],self.i),thresh_isl='%s'%(self.thresh_isl),thresh_pix='%s'%(self.thresh_pix),rms_box=(self.RMS_BOX[0],self.RMS_BOX[1]),rms_box_bright=(self.RMS_BOX_Bright[0],self.RMS_BOX_Bright[1]),adaptive_thresh=30,blank_limit=1E-4,atrous_do='True',ini_method='curvature')#,psf_vary_do='True',psf_stype_only='False',psf_snrcut=5,psf_snrcutstack=5,psf_snrtop=0.3)
#Skymodel
#write bbs Skymodel
img.write_catalog(outfile="""%sPybdsm_Skymodel_Iter%s"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='bbs',correct_proj='True',clobber='True')
#write ds9 Skymodel
img.write_catalog(outfile="""%sPybdsm_Skymodel_Iter%s.reg"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='ds9',correct_proj='True',clobber='True')
#Catalog
#write ds9 catalog
img.write_catalog(outfile="""%sCatalog_Iter%s.reg"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='ds9',correct_proj='True',clobber='True')
#write fits catalog
img.write_catalog(outfile="""%sCatalog_Iter%s.fits"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='fits',correct_proj='True',clobber='True')
#write fits catalog
#img.write_catalog(outfile="""%sCatalog_Iter%s"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='bbs',correct_proj='True',clobber='True')
# Store the path of the Skymodel and value to exploit
#Catalog path
#self.statisticsSkymodelCurrent = """%sCatalog_Iter%s"""%(self.SkymodelPath,self.i+1)
#if self.i !=0:
# self.statisticsSkymodelPrevious = """%sCatalog_Iter%s"""%(self.SkymodelPath,self.i)
# Store the path of the Skymodel and value to exploit
#Skymodel path
self.statisticsSkymodelCurrent = """%sPybdsm_Skymodel_Iter%s"""%(self.SkymodelPath,self.i+1)
if self.i !=0:
self.statisticsSkymodelPrevious = """%sPybdsm_Skymodel_Iter%s"""%(self.SkymodelPath,self.i)
# Values
self.rmsclipped = img.clipped_rms
self.Mean = img.clipped_mean
self.TotalFlux = img.total_flux_gaus
# convert dot model to BBS format
convert_cmd="""casapy2bbs.py --mask=%sMask_Iter%s.casa %sImage_%sarcsec_Iter%s.model.corr %sSkymodel_Iter%s"""%(self.SkymodelPath,self.i+1,self.ImagePathDir,self.pixsize[self.i],self.i,self.SkymodelPath,self.i+1)
print ''
print convert_cmd
print ''
os.system(convert_cmd)
if self.i == self.nbCycle:
#extract the source model with pybdsm
print ''
print 'extraction by pybdsm: bdsm.process_image %s,adaptive_rms_box=True,advanced_opts=True,detection_image=%s,thresh_isl=%s,thresh_pix=%s,rms_box=(%s,%s),rms_box_bright=(%s,%s),adaptive_thresh=30,blank_limit=1E-4,atrous_do=True,ini_method=curvature)#,psf_vary_do=True,psf_stype_only=False,psf_snrcut=5,psf_snrcutstack=5,psf_snrtop=0.3'%("""%sFinal_Image_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),"""%sFinal_Image_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),self.thresh_isl,self.thresh_pix,self.RMS_BOX[0],self.RMS_BOX[1],self.RMS_BOX_Bright[0],self.RMS_BOX_Bright[1])
print ''
#extract the source model with pybdsm
img = bdsm.process_image("""%sFinal_Image_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),adaptive_rms_box='True',advanced_opts='True',detection_image="""%sFinal_Image_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),thresh_isl='%s'%(self.thresh_isl),thresh_pix='%s'%(self.thresh_pix),rms_box=(self.RMS_BOX[0],self.RMS_BOX[1]),rms_box_bright=(self.RMS_BOX_Bright[0],self.RMS_BOX_Bright[1]),adaptive_thresh=30,blank_limit=1E-4,atrous_do='True',ini_method='curvature')#,psf_vary_do='True',psf_stype_only='False',psf_snrcut=5,psf_snrcutstack=5,psf_snrtop=0.3)
#Skymodel
#write bbs Skymodel
img.write_catalog(outfile="""%sFinal_Pybdsm_Skymodel_Iter%s"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='bbs',correct_proj='True',clobber='True')
#write ds9 Skymodel
img.write_catalog(outfile="""%sFinal_Pybdsm_Skymodel_Iter%s.reg"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='ds9',correct_proj='True',clobber='True')
#Catalog
#write ds9 catalog
img.write_catalog(outfile="""%sFinal_Catalog_Iter%s.reg"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='ds9',correct_proj='True',clobber='True')
#write fits catalog
img.write_catalog(outfile="""%sFinal_Catalog_Iter%s.fits"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='fits',correct_proj='True',clobber='True')
#write BBS catalog
#img.write_catalog(outfile="""%sFinal_Catalog_Iter%s"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='bbs',correct_proj='True',clobber='True')
# Store the path of the Skymodel and value to exploit
# Catalog path
#self.statisticsSkymodelCurrent = """%sFinal_Catalog_Iter%s"""%(self.SkymodelPath,self.i+1)
#self.statisticsSkymodelPrevious = """%sCatalog_Iter%s"""%(self.SkymodelPath,self.i)
# Store the path of the Skymodel and value to exploit
# Skymodel path
self.statisticsSkymodelCurrent = """%sFinal_Pybdsm_Skymodel_Iter%s"""%(self.SkymodelPath,self.i+1)
self.statisticsSkymodelPrevious = """%sPybdsm_Skymodel_Iter%s"""%(self.SkymodelPath,self.i)
# Values
self.rmsclipped = img.clipped_rms
self.Mean = img.clipped_mean
self.TotalFlux = img.total_flux_gaus
# convert dot model to BBS format
convert_cmd="""casapy2bbs.py --mask=%sFinal_Mask_Iter%s.casa %sFinal_Image_%sarcsec_Iter%s.model.corr %sFinal_Skymodel_Iter%s"""%(self.SkymodelPath,self.i+1,self.ImagePathDir,self.pixsize[self.i-1],self.i,self.SkymodelPath,self.i+1)
print ''
print convert_cmd
print ''
os.system(convert_cmd)
################################################################
# no use of mask
################################################################
else:
if self.i < self.nbCycle:
#extract the source model with pybdsm
print ''
print 'extraction by pybdsm: bdsm.process_image %s,adaptive_rms_box=True,advanced_opts=True,detection_image=%s,thresh_isl=%s,thresh_pix=%s,rms_box=(%s,%s),rms_box_bright=(%s,%s),adaptive_thresh=30,blank_limit=1E-4,atrous_do=True,ini_method=curvature)#,psf_vary_do=True,psf_stype_only=False,psf_snrcut=5,psf_snrcutstack=5,psf_snrtop=0.3'%("""%sImage_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i],self.i),"""%sImage_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i],self.i),self.thresh_isl,self.thresh_pix,self.RMS_BOX[0],self.RMS_BOX[1],self.RMS_BOX_Bright[0],self.RMS_BOX_Bright[1])
print ''
#extract the source model with pybdsm
img = bdsm.process_image("""%sImage_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i],self.i),adaptive_rms_box='True',advanced_opts='True',detection_image="""%sImage_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i],self.i),thresh_isl='%s'%(self.thresh_isl),thresh_pix='%s'%(self.thresh_pix),rms_box=(self.RMS_BOX[0],self.RMS_BOX[1]),rms_box_bright=(self.RMS_BOX_Bright[0],self.RMS_BOX_Bright[1]),adaptive_thresh=30,blank_limit=1E-4,atrous_do='True',ini_method='curvature')#,psf_vary_do='True',psf_stype_only='False',psf_snrcut=5,psf_snrcutstack=5,psf_snrtop=0.3)
#Skymodel
#write bbs catalog
img.write_catalog(outfile="""%sSkymodel_Iter%s"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='bbs',correct_proj='True',clobber='True')
#write bbs catalog
img.write_catalog(outfile="""%sSkymodel_Iter%s.reg"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='ds9',correct_proj='True',clobber='True')
#catalog
#write ds9 catalog
img.write_catalog(outfile="""%sCatalog_Iter%s.reg"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='ds9',correct_proj='True',clobber='True')
#write fits catalog
img.write_catalog(outfile="""%sCatalog_Iter%s.fits"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='fits',correct_proj='True',clobber='True')
#write bbs catalog
#img.write_catalog(outfile="""%sCatalog_Iter%s"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='bbs',correct_proj='True',clobber='True')
# Store the path of the Skymodel and value to exploit
# Catalog path
#self.statisticsSkymodelCurrent = """%sCatalog_Iter%s"""%(self.SkymodelPath,self.i+1)
#if self.i !=0:
# self.statisticsSkymodelPrevious = """%sCatalog_Iter%s"""%(self.SkymodelPath,self.i)
# Store the path of the Skymodel and value to exploit
# Skymodel path
self.statisticsSkymodelCurrent = """%sSkymodel_Iter%s"""%(self.SkymodelPath,self.i+1)
if self.i !=0:
self.statisticsSkymodelPrevious = """%sSkymodel_Iter%s"""%(self.SkymodelPath,self.i)
# Values
self.rmsclipped = img.clipped_rms
self.Mean = img.clipped_mean
self.TotalFlux = img.total_flux_gaus
if self.i == self.nbCycle:
#extract the source model with pybdsm
print ''
print 'extraction by pybdsm: bdsm.process_image %s,adaptive_rms_box=True,advanced_opts=True,detection_image=%s,thresh_isl=%s,thresh_pix=%s,rms_box=(%s,%s),rms_box_bright=(%s,%s),adaptive_thresh=30,blank_limit=1E-4,atrous_do=True,ini_method=curvature)#,psf_vary_do=True,psf_stype_only=False,psf_snrcut=5,psf_snrcutstack=5,psf_snrtop=0.3'%("""%sFinal_Image_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),"""%sFinal_Image_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),self.thresh_isl,self.thresh_pix,self.RMS_BOX[0],self.RMS_BOX[1],self.RMS_BOX_Bright[0],self.RMS_BOX_Bright[1])
print ''
#extract the source model with pybdsm
img = bdsm.process_image("""%sFinal_Image_%sarcsec_Iter%s.restored.corr"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),adaptive_rms_box='True',advanced_opts='True',detection_image="""%sFinal_Image_%sarcsec_Iter%s.restored"""%(self.ImagePathDir,self.pixsize[self.i-1],self.i),thresh_isl='%s'%(self.thresh_isl),thresh_pix='%s'%(self.thresh_pix),rms_box=(self.RMS_BOX[0],self.RMS_BOX[1]),rms_box_bright=(self.RMS_BOX_Bright[0],self.RMS_BOX_Bright[1]),adaptive_thresh=30,blank_limit=1E-4,atrous_do='True',ini_method='curvature')#,psf_vary_do='True',psf_stype_only='False',psf_snrcut=5,psf_snrcutstack=5,psf_snrtop=0.3)
#Skymodel
#write bbs catalog
img.write_catalog(outfile="""%sFinal_Skymodel_Iter%s"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='bbs',correct_proj='True',clobber='True')
#write bbs catalog
img.write_catalog(outfile="""%sFinal_Skymodel_Iter%s.reg"""%(self.SkymodelPath,self.i+1),catalog_type='gaul',format='ds9',correct_proj='True',clobber='True')
#Catalog
#write ds9 catalog
img.write_catalog(outfile="""%sFinal_Catalog_Iter%s.reg"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='ds9',correct_proj='True',clobber='True')
#write fits catalog
img.write_catalog(outfile="""%sFinal_Catalog_Iter%s.fits"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='fits',correct_proj='True',clobber='True')
#write bbs catalog
#img.write_catalog(outfile="""%sFinal_Catalog_Iter%s"""%(self.SkymodelPath,self.i+1),catalog_type='srl',format='bbs',correct_proj='True',clobber='True')
# Store the path of the Skymodel and value to exploit
# Skymodel path
#self.statisticsSkymodelCurrent = """%sFinal_Catalog_Iter%s"""%(self.SkymodelPath,self.i+1)
#self.statisticsSkymodelPrevious = """%sCatalog_Iter%s"""%(self.SkymodelPath,self.i)
# Store the path of the Skymodel and value to exploit
# Skymodel path
self.statisticsSkymodelCurrent = """%sFinal_Skymodel_Iter%s"""%(self.SkymodelPath,self.i+1)
self.statisticsSkymodelPrevious = """%sSkymodel_Iter%s"""%(self.SkymodelPath,self.i)
# Values
self.rmsclipped = img.clipped_rms
self.Mean = img.clipped_mean
self.TotalFlux = img.total_flux_gaus
def main(image_name,
mask_name,
image_beam=False,
atrous_do=False,
threshisl=0.0,
threshpix=0.0,
rmsbox=None,
iterate_threshold=False,
adaptive_rmsbox=False,
beam=None,
img_format=None):
convbox = rmsbox.rstrip(')').lstrip('(').split(',')
rmsbox = (int(convbox[0]), int(convbox[1]))
if image_beam:
print 'do stuff for the beam'
if atrous_do:
threshisl = 4.0
if iterate_threshold:
# Start with high threshold and lower it until we get at least one island
threshpix_orig = threshpix
threshisl_orig = threshisl
nisl = 0
threshpix = 25
threshisl = 15
while nisl == 0:
img = bdsm.process_image(image_name,
mean_map='zero',
rms_box=rmsbox,
thresh_pix=numpy.float(threshpix),
thresh_isl=numpy.float(threshisl),
atrous_do=atrous_do,
ini_method='curvature',
beam=beam,
adaptive_rms_box=adaptive_rmsbox,
adaptive_thresh=20,
quiet=True)
nisl = img.nisl
threshpix /= 1.2
threshisl /= 1.2
threshpix = threshpix_orig
threshisl = threshisl_orig
else:
img = bdsm.process_image(image_name,
mean_map='zero',
rms_box=rmsbox,
thresh_pix=numpy.float(threshpix),
thresh_isl=numpy.float(threshisl),
atrous_do=atrous_do,
ini_method='curvature',
beam=beam,
adaptive_rms_box=adaptive_rmsbox,
adaptive_thresh=20,
quiet=True)
img.export_image(img_type='island_mask',
mask_dilation=0,
outfile=mask_name,
img_format=img_format,
clobber=True)
log_file = mask_name + '.log'
with open(log_file, 'wb') as f:
f.write('# 5-sigma clipped rms (Jy/beam): {0}'.format(5.0 *
img.clipped_rms))
image_name = args[0]
mask_name = image_name.split('-image.fits')[0] + '.fitsmask'
print '\n\n\n'
print 'Making mask:', mask_name
print '\n\n\n'
os.system('rm -rf ' + mask_name)
# DO THE SOURCE DETECTION
img = bdsm.process_image(image_name, mean_map='zero', rms_box=(60,10), thresh_pix=numpy.float(o.threshpix), \
thresh_isl=numpy.float(o.threshisl), atrous_do=o.atrous_do, \
adaptive_rms_box=False, adaptive_thresh=150, rms_box_bright=(60,10), atrous_jmax=3)
#img.show_fit()
# WRITE THE MASK FITS
img.export_image(img_type='island_mask', img_format='fits', outfile=mask_name)
# convert mask to casapy format
#sys.exit()
#img = pyrap.images.image(mask_name)
#pixels = numpy.copy(img.getdata())
#pixels_mask = 0.*numpy.copy(pixels)
for image_nvss in images_nvss:
image_tgss = image_nvss.replace('NVSS','TGSS')
if not os.path.exists(image_tgss):
print "TGSS file: ", image_tgss, "does not exist, continue."
continue
print "-- Make catlog on:", image_nvss, image_tgss
# source finder
image_rms_nvss = image_nvss.replace('.fits','-rms.fits').replace('NVSS','NVSS/rms',1)
image_gaus_nvss = image_nvss.replace('.fits','-gaus.fits').replace('NVSS','NVSS/gaus',1)
image_isl_nvss = image_nvss.replace('.fits','-isl.fits').replace('NVSS','NVSS/isl',1)
cat_srl_nvss = image_nvss.replace('.fits','-srl.fits').replace('NVSS','NVSS/catalog',1)
if not os.path.exists(cat_srl_nvss) or not os.path.exists(image_isl_nvss) or not os.path.exists(image_rms_nvss) or not os.path.exists(image_gaus_nvss):
c = bdsm.process_image(image_nvss, frequency=1400e6, rms_box=(102,34), advanced_opts=True, group_tol=0.5, thresh_isl=3, thresh_pix=4)
c.export_image(outfile=image_rms_nvss, img_type='rms', clobber=True)
c.export_image(outfile=image_gaus_nvss, img_type='gaus_model', clobber=True)
c.export_image(outfile=image_isl_nvss, img_type='island_mask', clobber=True)
c.write_catalog(outfile=cat_srl_nvss, catalog_type='srl', format='fits', clobber=True)
# clip the gaus images and create island masks
# we use gaus images because they catch better the extended emission than isl_mask
clip_gaus(image_gaus_nvss, image_rms_nvss)
image_rms_tgss = image_tgss.replace('.fits','-rms.fits').replace('TGSS','TGSS/rms',1)
image_gaus_tgss = image_tgss.replace('.fits','-gaus.fits').replace('TGSS','TGSS/gaus',1)
image_isl_tgss = image_tgss.replace('.fits','-isl.fits').replace('TGSS','TGSS/isl',1)
cat_srl_tgss = image_tgss.replace('.fits','-srl.fits').replace('TGSS','TGSS/catalog',1)
if not os.path.exists(cat_srl_tgss) or not os.path.exists(image_isl_tgss) or not os.path.exists(image_rms_tgss) or not os.path.exists(image_gaus_tgss):
c = bdsm.process_image(image_tgss, frequency=147e6, rms_box=(102,34), advanced_opts=True, group_tol=0.5, thresh_isl=3, thresh_pix=4)
c.export_image(outfile=image_rms_tgss, img_type='rms', clobber=True)
def pybdsm_search(image="${imager.RESTORED_IMAGE}",
output="$PYBDSM_OUTPUT",
pol='$PYBDSM_POLARIZED',
select=None,
center=None,
threshold=None,
pbexp=None,
**kw):
"""Runs pybdsm on the specified 'image', converts the results into a Tigger model and writes it to 'output'.
Use 'threshold' to specify a non-default threshold (thresh_isl and thresh_pix).
Use 'pol' to force non-default polarized mode.
Use 'pbexp' to supply a primary beam expression (passed to tigger-convert), in which case the output model will contain
intrinsic fluxes.
Use 'select' to apply a selection string on the new model (e.g. "I.gt.0.001")
Use 'center' to set the centre of the model (useful for e.g. selecting on radius), use a string e.g. Xdeg,Ydeg
"""
image, output, pol, center = interpolate_locals("image output pol center")
makedir(v.DESTDIR)
# setup parameters
gaul = II("${output:BASEPATH}.gaul")
# info("PyBDSM filenames are $output $gaul");
# start with default PYBDSM options
opts = PYBDSM_OPTIONS.copy()
opts.update(kw)
# override with explicit arguments
if threshold:
opts['thresh_pix'] = threshold
if pol is not None:
opts['polarisation_do'] = is_true(pol)
pol = opts.get('polarisation_do', False)
opts['quiet'] = True
# run pybdsm
info("running PyBDSM process_image($image,%s)" %
",".join(sorted(["%s=%s" % x for x in opts.items()])))
from lofar import bdsm
img = bdsm.process_image(image, **opts)
info("writing PyBDSM gaul catalog")
img.write_catalog(outfile=gaul,
format='ascii',
catalog_type='gaul',
clobber=True)
# add log to output
logfile = II("${output:BASEPATH}.pybdsm.log")
if exists(logfile):
info("PyBDSM log output follows:")
for line in file(logfile):
print(" ", line)
else:
warn("PyBDSM log $logfile not found")
# set clustering parameter from beam size
cluster = CLUSTER_DIST
if not cluster:
hdr = pyfits.open(image)[0].header
# BMAJ/BMIN is in degrees -- convert to seconds, or fall back to 60" if not set
cluster = 1800 * (hdr.get('BMAJ', 0) +
hdr.get('BMIN', 0)) * CLUSTER_DIST_BEAMS or 60
# convert catalog
if pbexp:
args = ["--primary-beam", pbexp, "--app-to-int"]
else:
args = []
if select:
args += ["--select", select]
if center:
args += ["--center", center]
## leaving this for now but it seems overly complicated. Better to add an option
## to tigger-convert to eliminate sources with NaN positions
verifyGaulModel(gaul)
tigger_convert(gaul,
output,
"-t",
"Gaul",
"-f",
"--rename",
"--cluster-dist",
cluster,
"--min-extent",
MIN_EXTENT,
split_args=False,
*args)
workingimage= 'IMAGE_%s_%s'%(round(resolution,2),i)
if i == 0:
iterations,threshold = 300000,'%smJy'%(np.max([infodict['Est Noise']*10.0,brightestsource*1000.0/400.0]))
prevthreshold = np.max([infodict['Est Noise']*10.0,brightestsource*1000.0/400.0])
else:
iterations,threshold = 300000,'%smJy'%(np.max([rms*8.0*1000,brightestsource*1000.0/400.0])) # Do not clean so deeply without the clean boxes
prevthreshold = np.max([rms*8.0*1000,brightestsource*1000.0/400.0])
flag_measurementset(new_lofar_imagemsfile,new_lofar_imagemsfile,'CORRECTED_DATA','ndppp_flag_%s.log'%i)
aw_imagedata('%s'%new_lofar_imagemsfile,'aw_imager_%s.inp'%i,'%s'%workingimage,iterations,threshold,cellsize,uvmin,uvmax,npix,'mfclark','',robust)
os.system("echo 'Iter%s initial imaging time %s \n' >> %s"%(i,time.time()-previoustime,loggingfilename))
previoustime = time.time()
img = bdsm.process_image('%s.restored'%workingimage,advanced_opts='True',detection_image='%s.restored'%workingimage,thresh_isl=3,thresh_pix=5,blank_limit=1E-4,adaptive_rms_box='True',adaptive_thresh=200)#,adaptive_rms_box='True',atrous_do='True')
img.export_image(outfile="mask_%s"%workingimage,img_type='island_mask',img_format='casa')
img.export_image(outfile="rms_%s"%workingimage,img_type='rms',img_format='casa')
os.system('image2fits in=%s.restored out=%s.restored.fits'%(workingimage,workingimage))
f = pyfits.open('%s.restored.fits'%workingimage)
noisearray = f[0].data.flatten()
maxpixel = np.max(noisearray)
noisearray = np.random.permutation(noisearray)[:10000]
noisepix = np.array(filter(lambda x: abs(x) > 10E-8,noisearray))
noisepix = np.array(filter(lambda x: abs(x)<infodict['Est Noise']*50.0/1000.0,noisepix))
rms = fit_gaussian_histogram(noisepix,'n')
print 'rms %s, maxpixel %s'%(rms,maxpixel)
f.close()
minthreshold = rms
image = pim.image("rms_%s"%workingimage)
import lofar.bdsm as bdsm
import sys
# Process the image
img = bdsm.process_image('tbdsm_process_image.in')
# List of operations that must have been done on `img`.
operations = [
'readimage', 'collapse', 'preprocess', 'rmsimage', 'threshold',
'islands', 'gausfit', 'gaul2srl', 'make_residimage', 'wavelet_atrous',
'shapelets', 'spectralindex', 'polarisation', 'psf_vary', 'cleanup'
]
# Return exit status 0 if everything went fine, otherwise return 1.
if img and all(oper in img.completed_Ops for oper in operations):
sys.exit(0)
else:
sys.exit(1)
if options.noscreen:
imageroots.append('original')
use_ions.append(False)
UVmax = options.uvmax
for imageroot, use_ion in zip(imageroots, use_ions):
log.info('Calling AWimager to make {0} image...'.format(imageroot))
if options.mask:
from lofar import bdsm
mask_image = imagedir + '/' + imageroot + '.mask'
log.info('Generating mask "{0}"...'.format(mask_image))
awimager(msname, imageroot, UVmax, options.size, options.npix,
options.threshold*5.0, clobber=options.clobber,
use_ion=use_ion, imagedir=imagedir)
img = bdsm.process_image(imagedir+'/'+imageroot+'.restored',
blank_limit=1e-4, stop_at='isl', thresh_pix=6,
thresh_isl=4)
img.export_image(outfile=mask_image, img_type='island_mask',
img_format='casa', mask_dilation=2, clobber=True)
threshold = img.clipped_rms * 5.0
log.info('Cleaning to threshold of {0} Jy...'.format(threshold))
awimager(msname, imageroot, UVmax, options.size, options.npix,
threshold, mask_image=mask_image, use_ion=use_ion,
imagedir=imagedir, logfilename=logfilename, clobber=True)
else:
log.info('Cleaning to threshold of {0} Jy...'.format(threshold))
awimager(msname, imageroot, UVmax, options.size, options.npix,
options.threshold, clobber=options.clobber, use_ion=use_ion,
imagedir=imagedir, logfilename=logfilename)
log.info('Imaging complete.')
def run(self, input_image, bdsm_parameter_run1_path,
bdsm_parameter_run2x_path, catalog_output_path, image_output_path,
sourcedb_target_path, environment, working_directory,
create_sourcdb_exec):
"""
:param input_image: image to look for sources in
:param bdsm_parameter_run1_path: parset with bdsm parameters for the
first run
:param bdsm_parameter_run2x_path: second ron bdsm parameters
:param catalog_output_path: Path to full list of sources found
:param image_output_path: Path to fits image with all sources
substracted
:param sourcedb_target_path: Path to store the sourcedb created from
containing all the found sources
:param environment: environment for runwithlog4cplus
:param working_directory: Working dir
:param create_sourcdb_exec: Path to create sourcedb executable
:rtype: self.outputs['source_db'] sourcedb_target_path
"""
#******************************************************************
# 0. Create the directories used in this recipe
create_directory(working_directory)
import lofar.bdsm as bdsm#@UnresolvedImport
self.logger.info("Starting imager_source_finding")
self.environment.update(environment)
# default frequency is None (read from image), save for later cycles.
# output of pybdsm forgets freq of source image
frequency = None
# Output of the for loop: n iterations and any source found
n_itter_sourcefind = None
sources_found = False
max_sourcefind_itter = 5 # TODO: maximum itter is a magic value
for idx in range(max_sourcefind_itter):
# ******************************************************************
# 1. Select correct input image
# The first iteration uses the input image, second and later use the
# output of the previous iteration. The 1+ iteration have a
# seperate parameter set.
if idx == 0:
input_image_local = input_image # input_image_cropped
image_output_path_local = image_output_path + "_0"
bdsm_parameter_local = parameterset(bdsm_parameter_run1_path)
else:
input_image_local = image_output_path + "_{0}".format(
str(idx - 1))
image_output_path_local = image_output_path + "_{0}".format(
str(idx))
bdsm_parameter_local = parameterset(bdsm_parameter_run2x_path)
# *****************************************************************
# 2. parse the parameters and convert to python if possible
# this is needed for pybdsm
bdsm_parameters = {}
for key in bdsm_parameter_local.keys():
parameter_value = bdsm_parameter_local.getStringVector(key)[0]
try:
parameter_value = eval(parameter_value)
except:
pass #do nothing
bdsm_parameters[key] = parameter_value
# pybdsm needs its filename here, to derive the log location
bdsm_parameters["filename"] = input_image_local
# *****************************************************************
# 3. Start pybdsm
self.logger.debug(
"Starting sourcefinder bdsm on {0} using parameters:".format(
input_image_local))
self.logger.debug(repr(bdsm_parameters))
img = bdsm.process_image(bdsm_parameters, frequency = frequency)
# Always export the catalog
img.write_catalog(
outfile = catalog_output_path + "_{0}".format(str(idx)),
catalog_type = 'gaul', clobber = True,
format = "bbs", force_output = True)
# If no more matching of sources with gausians is possible (nsrc==0)
# break the loop
if img.nsrc == 0:
n_itter_sourcefind = idx
break
# We have at least found a single source!
self.logger.debug("Number of source found: {0}".format(
img.nsrc))
# *****************************************************************
# 4. export the image
self.logger.debug("Wrote list of sources to file at: {0})".format(
catalog_output_path))
img.export_image(outfile = image_output_path_local,
img_type = 'gaus_resid', clobber = True,
img_format = "fits")
self.logger.debug("Wrote fits image with substracted sources"
" at: {0})".format(image_output_path_local))
# Save the frequency from image header of the original input file,
# This information is not written by pybdsm to the exported image
frequency = img.frequency
# if not set the maximum number of itteration us performed
if n_itter_sourcefind == None:
n_itter_sourcefind = max_sourcefind_itter
# ********************************************************************
# 5. The produced catalogs now need to be combined into a single list
# Call with the number of loops and the path to the files, only combine
# if we found sources
self.logger.debug(
"Writing source list to file: {0}".format(catalog_output_path))
self._combine_source_lists(n_itter_sourcefind, catalog_output_path)
# *********************************************************************
# 6. Convert sourcelist to sourcedb
self._create_source_db(catalog_output_path, sourcedb_target_path,
working_directory, create_sourcdb_exec, False)
# Assign the outputs
self.outputs["catalog_output_path"] = catalog_output_path
self.outputs["source_db"] = sourcedb_target_path
return 0
t = pim.image(glob.glob('SB{:03d}*residual.corr'.format(sb))[0])
t.tofits('temp.fits')
os.system('fits op=xyin in=temp.fits out=temp')
os.system('puthd in=temp/bmaj value={},"arcseconds"'.format(bpar_ma))
os.system('puthd in=temp/bmin value={},"arcseconds"'.format(bpar_mi))
os.system('puthd in=temp/bpa value={},"degrees"'.format(bpar_pa))
os.system('puthd in=temp/bunit value="Jy/Beam"')
os.system('prthd in=temp')
os.system('convol map=temp options=final fwhm={} out=temp.conv'.format(smoothSize))
os.system('fits op=xyout in=temp.conv out={}'.format(residualCorr))
os.system('rm -r temp temp.conv t.fits')
# Run pybdsm on the residual images
residualMask = 'mask_{:03d}.residual.fits'.format(sb)
img = bdsm.process_image(residual, adaptive_rms_box=True, advanced_opts=True, \
detection_image=residualCorr, ini_method='curvature', atrous_do=True, \
psf_vary_do=False, thresh_isl=6.0, thresh_pix=8.0, \
beam=(smoothSize/3600.,smoothSize/3600.,0.))
img.export_image(outfile=residualMask, img_type='island_mask', img_format='fits', clobber=True)
img.export_image(outfile=residualMask+'.casa', img_type='island_mask', img_format='casa', clobber=True)
# Run pybdsm on the residual images
restoredMask = 'mask_{:03d}.restored.fits'.format(sb)
img = bdsm.process_image(restored, adaptive_rms_box=True, advanced_opts=True, \
detection_image=restoredCorr, ini_method='curvature', atrous_do=True, \
psf_vary_do=True, thresh_isl=6.0, thresh_pix=8.0)
img.export_image(outfile=restoredMask, img_type='island_mask', img_format='fits', clobber=True)
img.export_image(outfile=restoredMask+'.casa', img_type='island_mask', img_format='casa', clobber=True)
# Merge the residual and the restored masks
a = pim.image(restoredMask+'.casa')
b = pim.image(residualMask+'.casa')
def main(image_name, mask_name, atrous_do=False, threshisl=0.0, threshpix=0.0, rmsbox=None,
rmsbox_bright=(35, 7), iterate_threshold=False, adaptive_rmsbox=False, img_format='fits',
threshold_format='float', trim_by=0.0, vertices_file=None, atrous_jmax=6,
pad_to_size=None, skip_source_detection=False, region_file=None, nsig=1.0,
reference_ra_deg=None, reference_dec_deg=None, cellsize_deg=0.000417,
use_adaptive_threshold=False):
"""
Make a clean mask and return clean threshold
Parameters
----------
image_name : str
Filename of input image from which mask will be made. If the image does
not exist, a template image with center at (reference_ra_deg,
reference_dec_deg) will be made internally
mask_name : str
Filename of output mask image
atrous_do : bool, optional
Use wavelet module of PyBDSM?
threshisl : float, optional
Value of thresh_isl PyBDSM parameter
threshpix : float, optional
Value of thresh_pix PyBDSM parameter
rmsbox : tuple of floats, optional
Value of rms_box PyBDSM parameter
rmsbox_bright : tuple of floats, optional
Value of rms_box_bright PyBDSM parameter
iterate_threshold : bool, optional
If True, threshold will be lower in 20% steps until
at least one island is found
adaptive_rmsbox : tuple of floats, optional
Value of adaptive_rms_box PyBDSM parameter
img_format : str, optional
Format of output mask image (one of 'fits' or 'casa')
threshold_format : str, optional
Format of output threshold (one of 'float' or 'str_with_units')
trim_by : float, optional
Fraction by which the perimeter of the output mask will be
trimmed (zeroed)
vertices_file : str, optional
Filename of file with vertices (must be a pickle file containing
a dictionary with the vertices in the 'vertices' entry)
atrous_jmax : int, optional
Value of atrous_jmax PyBDSM parameter
pad_to_size : int, optional
Pad output mask image to a size of pad_to_size x pad_to_size
skip_source_detection : bool, optional
If True, source detection is not run on the input image
region_file : str, optional
Filename of region file in CASA format. If given, no mask image
is made (the region file is used as the clean mask)
nsig : float, optional
Number of sigma of returned threshold value
reference_ra_deg : float, optional
RA for center of output mask image
reference_dec_deg : float, optional
Dec for center of output mask image
cellsize_deg : float, optional
Size of a pixel in degrees
use_adaptive_threshold : bool, optional
If True, use an adaptive threshold estimated from the negative values in
the image
Returns
-------
result : dict
Dict with nsig-sigma rms threshold
"""
if rmsbox is not None and type(rmsbox) is str:
rmsbox = eval(rmsbox)
if type(rmsbox_bright) is str:
rmsbox_bright = eval(rmsbox_bright)
if pad_to_size is not None and type(pad_to_size) is str:
pad_to_size = int(pad_to_size)
if type(atrous_do) is str:
if atrous_do.lower() == 'true':
atrous_do = True
threshisl = 4.0 # override user setting to ensure proper source fitting
else:
atrous_do = False
if type(iterate_threshold) is str:
if iterate_threshold.lower() == 'true':
iterate_threshold = True
else:
iterate_threshold = False
if type(adaptive_rmsbox) is str:
if adaptive_rmsbox.lower() == 'true':
adaptive_rmsbox = True
else:
adaptive_rmsbox = False
if type(skip_source_detection) is str:
if skip_source_detection.lower() == 'true':
skip_source_detection = True
else:
skip_source_detection = False
if type(use_adaptive_threshold) is str:
if use_adaptive_threshold.lower() == 'true':
use_adaptive_threshold = True
else:
use_adaptive_threshold = False
if reference_ra_deg is not None and reference_dec_deg is not None:
reference_ra_deg = float(reference_ra_deg)
reference_dec_deg = float(reference_dec_deg)
if not os.path.exists(image_name):
print('Input image not found. Making empty image...')
if not skip_source_detection:
print('ERROR: Source detection cannot be done on an empty image')
sys.exit(1)
if reference_ra_deg is not None and reference_dec_deg is not None:
image_name = mask_name + '.tmp'
make_template_image(image_name, reference_ra_deg, reference_dec_deg,
cellsize_deg=float(cellsize_deg))
else:
print('ERROR: if image not found, a refernce position must be given')
sys.exit(1)
trim_by = float(trim_by)
atrous_jmax = int(atrous_jmax)
threshpix = float(threshpix)
threshisl = float(threshisl)
nsig = float(nsig)
threshold = 0.0
if not skip_source_detection:
if vertices_file is not None:
# Modify the input image to blank the regions outside of the polygon
temp_img = pim.image(image_name)
image_name += '.blanked'
temp_img.saveas(image_name, overwrite=True)
input_img = pim.image(image_name)
data = input_img.getdata()
vertices = read_vertices(vertices_file)
RAverts = vertices[0]
Decverts = vertices[1]
xvert = []
yvert = []
for RAvert, Decvert in zip(RAverts, Decverts):
pixels = input_img.topixel([1, 1, Decvert*np.pi/180.0,
RAvert*np.pi/180.0])
xvert.append(pixels[2]) # x -> Dec
yvert.append(pixels[3]) # y -> RA
poly = Polygon(xvert, yvert)
# Find masked regions
masked_ind = np.where(data[0, 0])
# Find distance to nearest poly edge and set to NaN those that
# are outside the facet (dist < 0)
dist = poly.is_inside(masked_ind[0], masked_ind[1])
outside_ind = np.where(dist < 0.0)
if len(outside_ind[0]) > 0:
data[0, 0, masked_ind[0][outside_ind], masked_ind[1][outside_ind]] = np.nan
# Save changes
input_img.putdata(data)
if use_adaptive_threshold:
# Get an estimate of the rms
img = bdsm.process_image(image_name, mean_map='zero', rms_box=rmsbox,
thresh_pix=threshpix, thresh_isl=threshisl,
atrous_do=atrous_do, ini_method='curvature', thresh='hard',
adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=150,
rms_box_bright=rmsbox_bright, rms_map=True, quiet=True,
atrous_jmax=atrous_jmax, stop_at='isl')
# Find min and max pixels
max_neg_val = abs(np.min(img.ch0_arr))
max_neg_pos = np.where(img.ch0_arr == np.min(img.ch0_arr))
max_pos_val = abs(np.max(img.ch0_arr))
max_pos_pos = np.where(img.ch0_arr == np.max(img.ch0_arr))
# Estimate new thresh_isl from min pixel value's sigma, but don't let
# it get higher than 1/2 of the peak's sigma
threshisl_neg = 2.0 * max_neg_val / img.rms_arr[max_neg_pos][0]
max_sigma = max_pos_val / img.rms_arr[max_pos_pos][0]
if threshisl_neg > max_sigma / 2.0:
threshisl_neg = max_sigma / 2.0
# Use the new threshold only if it is larger than the user-specified one
if threshisl_neg > threshisl:
threshisl = threshisl_neg
if iterate_threshold:
# Start with given threshold and lower it until we get at least one island
nisl = 0
while nisl == 0:
img = bdsm.process_image(image_name, mean_map='zero', rms_box=rmsbox,
thresh_pix=threshpix, thresh_isl=threshisl,
atrous_do=atrous_do, ini_method='curvature', thresh='hard',
adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=150,
rms_box_bright=rmsbox_bright, rms_map=True, quiet=True,
atrous_jmax=atrous_jmax)
nisl = img.nisl
threshpix /= 1.2
threshisl /= 1.2
if threshpix < 5.0:
break
else:
img = bdsm.process_image(image_name, mean_map='zero', rms_box=rmsbox,
thresh_pix=threshpix, thresh_isl=threshisl,
atrous_do=atrous_do, ini_method='curvature', thresh='hard',
adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=150,
rms_box_bright=rmsbox_bright, rms_map=True, quiet=True,
atrous_jmax=atrous_jmax)
if img.nisl == 0:
if region_file is None or region_file == '[]':
print('No islands found. Clean mask cannot be made.')
sys.exit(1)
else:
# Continue on and use user-supplied region file
skip_source_detection = True
threshold = nsig * img.clipped_rms
# Check if there are large islands preset (indicating that multi-scale
# clean is needed)
has_large_isl = False
for isl in img.islands:
if isl.size_active > 100:
# Assuming normal sampling, a size of 100 pixels would imply
# a source of ~ 10 beams
has_large_isl = True
if (region_file is not None and region_file != '[]' and skip_source_detection):
# Copy region file and return if source detection was not done
os.system('cp {0} {1}'.format(region_file.strip('[]"'), mask_name))
if threshold_format == 'float':
return {'threshold_5sig': threshold}
elif threshold_format == 'str_with_units':
# This is done to get around the need for quotes around strings in casapy scripts
# 'casastr/' is removed by the generic pipeline
return {'threshold_5sig': 'casastr/{0}Jy'.format(threshold)}
elif not skip_source_detection:
img.export_image(img_type='island_mask', mask_dilation=0, outfile=mask_name,
img_format=img_format, clobber=True)
if (vertices_file is not None or trim_by > 0 or pad_to_size is not None
or (region_file is not None and region_file != '[]')
or skip_source_detection):
# Alter the mask in various ways
if skip_source_detection:
# Read the image
mask_im = pim.image(image_name)
else:
# Read the PyBDSM mask
mask_im = pim.image(mask_name)
data = mask_im.getdata()
coordsys = mask_im.coordinates()
if reference_ra_deg is not None and reference_dec_deg is not None:
values = coordsys.get_referencevalue()
values[2][0] = reference_dec_deg/180.0*np.pi
values[2][1] = reference_ra_deg/180.0*np.pi
coordsys.set_referencevalue(values)
imshape = mask_im.shape()
del(mask_im)
if pad_to_size is not None:
imsize = pad_to_size
coordsys['direction'].set_referencepixel([imsize/2, imsize/2])
pixmin = (imsize - imshape[2]) / 2
if pixmin < 0:
print("The padded size must be larger than the original size.")
sys.exit(1)
pixmax = pixmin + imshape[2]
data_pad = np.zeros((1, 1, imsize, imsize), dtype=np.float32)
data_pad[0, 0, pixmin:pixmax, pixmin:pixmax] = data[0, 0]
new_mask = pim.image('', shape=(1, 1, imsize, imsize), coordsys=coordsys)
new_mask.putdata(data_pad)
else:
new_mask = pim.image('', shape=imshape, coordsys=coordsys)
new_mask.putdata(data)
data = new_mask.getdata()
if skip_source_detection:
# Mask all pixels
data[:] = 1
if vertices_file is not None:
# Modify the clean mask to exclude regions outside of the polygon
vertices = read_vertices(vertices_file)
RAverts = vertices[0]
Decverts = vertices[1]
xvert = []
yvert = []
for RAvert, Decvert in zip(RAverts, Decverts):
try:
pixels = new_mask.topixel([0, 1, Decvert*np.pi/180.0,
RAvert*np.pi/180.0])
except:
pixels = new_mask.topixel([1, 1, Decvert*np.pi/180.0,
RAvert*np.pi/180.0])
xvert.append(pixels[2]) # x -> Dec
yvert.append(pixels[3]) # y -> RA
poly = Polygon(xvert, yvert)
# Find masked regions
masked_ind = np.where(data[0, 0])
# Find distance to nearest poly edge and unmask those that
# are outside the facet (dist < 0)
dist = poly.is_inside(masked_ind[0], masked_ind[1])
outside_ind = np.where(dist < 0.0)
if len(outside_ind[0]) > 0:
data[0, 0, masked_ind[0][outside_ind], masked_ind[1][outside_ind]] = 0
if trim_by > 0.0:
sh = np.shape(data)
margin = int(sh[2] * trim_by / 2.0 )
data[0, 0, 0:sh[2], 0:margin] = 0
data[0, 0, 0:margin, 0:sh[3]] = 0
data[0, 0, 0:sh[2], sh[3]-margin:sh[3]] = 0
data[0, 0, sh[2]-margin:sh[2], 0:sh[3]] = 0
if region_file is not None and region_file != '[]':
# Merge the CASA regions with the mask
casa_polys = read_casa_polys(region_file.strip('[]"'), new_mask)
for poly in casa_polys:
# Find unmasked regions
unmasked_ind = np.where(data[0, 0] == 0)
# Find distance to nearest poly edge and mask those that
# are inside the casa region (dist > 0)
dist = poly.is_inside(unmasked_ind[0], unmasked_ind[1])
inside_ind = np.where(dist > 0.0)
if len(inside_ind[0]) > 0:
data[0, 0, unmasked_ind[0][inside_ind], unmasked_ind[1][inside_ind]] = 1
# Save changes
new_mask.putdata(data)
if img_format == 'fits':
new_mask.tofits(mask_name, overwrite=True)
elif img_format == 'casa':
new_mask.saveas(mask_name, overwrite=True)
else:
print('Output image format "{}" not understood.'.format(img_format))
sys.exit(1)
if not skip_source_detection:
if threshold_format == 'float':
return {'threshold_5sig': nsig * img.clipped_rms, 'multiscale': has_large_isl}
elif threshold_format == 'str_with_units':
# This is done to get around the need for quotes around strings in casapy scripts
# 'casastr/' is removed by the generic pipeline
return {'threshold_5sig': 'casastr/{0}Jy'.format(nsig * img.clipped_rms),
'multiscale': has_large_isl}
else:
return {'threshold_5sig': '0.0'}
def pybdsm_search (image="${imager.RESTORED_IMAGE}",output="$PYBDSM_OUTPUT",pol='$PYBDSM_POLARIZED',
select=None,
threshold=None,pbexp=None,**kw):
"""Runs pybdsm on the specified 'image', converts the results into a Tigger model and writes it to 'output'.
Use 'threshold' to specify a non-default threshold (thresh_isl and thresh_pix).
Use 'pol' to force non-default polarized mode.
Use 'pbexp' to supply a primary beam expression (passed to tigger-convert), in which case the output model will contain
intrinsic fluxes.
Use 'select' to apply a selection string on the new model (e.g. "I.gt.0.001")
"""
image,output,pol = interpolate_locals("image output pol");
makedir(v.DESTDIR);
# setup parameters
gaul = II("${output:BASEPATH}.gaul");
# info("PyBDSM filenames are $output $gaul");
# start with default PYBDSM options
opts = PYBDSM_OPTIONS.copy();
opts.update(kw);
# override with explicit arguments
if threshold:
opts['thresh_pix'] = threshold;
if pol is not None:
opts['polarisation_do'] = is_true(pol);
pol = opts.get('polarisation_do',False);
opts['quiet'] = True;
# run pybdsm
info("running PyBDSM process_image($image,%s)"%",".join(sorted([ "%s=%s"%x for x in opts.iteritems() ])));
from lofar import bdsm
img = bdsm.process_image(image,**opts);
info("writing PyBDSM gaul catalog");
img.write_catalog(outfile=gaul,format='ascii',catalog_type='gaul',clobber=True);
# add log to output
logfile = II("${output:BASEPATH}.pybdsm.log");
if exists(logfile):
info("PyBDSM log output follows:");
for line in file(logfile):
print " ",line;
else:
warn("PyBDSM log $logfile not found");
# set clustering parameter from beam size
cluster = CLUSTER_DIST;
if not cluster:
hdr = pyfits.open(image)[0].header;
# BMAJ/BMIN is in degrees -- convert to seconds, or fall back to 60" if not set
cluster = 1800*(hdr.get('BMAJ',0)+hdr.get('BMIN',0))*CLUSTER_DIST_BEAMS or 60;
# convert catalog
if pbexp:
args = [ "--primary-beam",pbexp,"--app-to-int" ]
else:
args = []
if select:
args += [ "--select",select ];
verifyGaulModel(gaul)
#Dictionary for establishing correspondence between parameter names in gaul files produced by pybdsm, and pyxis parameter names
dict_gaul2lsm = {'Gaus_id':'name', 'Isl_id':'Isl_id', 'Source_id':'Source_id', 'Wave_id':'Wave_id', 'RA':'ra_d', 'E_RA':'E_RA', 'DEC':'dec_d', 'E_DEC':'E_DEC', 'Total_flux':'i', 'E_Total_flux':'E_Total_flux', 'Peak_flux':'Peak_flux', 'E_Peak_flux':'E_Peak_flux', 'Xposn':'Xposn', 'E_Xposn':'E_Xposn', 'Yposn':'Yposn', 'E_Yposn':'E_Yposn', 'Maj':'Maj', 'E_Maj':'E_Maj', 'Min':'Min', 'E_Min':'E_Min', 'PA':'PA', 'E_PA':'E_PA', 'Maj_img_plane':'Maj_img_plane', 'E_Maj_img_plane':'E_Maj_img_plane', 'Min_img_plane':'Min_img_plane', 'E_Min_img_plane':'E_Min_img_plane', 'PA_img_plane':'PA_img_plane', 'E_PA_img_plane':'E_PA_img_plane', 'DC_Maj':'emaj_d', 'E_DC_Maj':'E_DC_Maj', 'DC_Min':'emin_d', 'E_DC_Min':'E_DC_Min', 'DC_PA':'pa_d', 'E_DC_PA':'E_DC_PA', 'DC_Maj_img_plane':'DC_Maj_img_plane', 'E_DC_Maj_img_plane':'E_DC_Maj_img_plane', 'DC_Min_img_plane':'DC_Min_img_plane', 'E_DC_Min_img_plane':'E_DC_Min_img_plane', 'DC_PA_img_plane':'DC_PA_img_plane', 'E_DC_PA_img_plane':'E_DC_PA_img_plane', 'Isl_Total_flux':'Isl_Total_flux', 'E_Isl_Total_flux':'E_Isl_Total_flux', 'Isl_rms':'Isl_rms', 'Isl_mean':'Isl_mean', 'Resid_Isl_rms':'Resid_Isl_rms', 'Resid_Isl_mean':'Resid_Isl_mean', 'S_Code':'S_Code', 'Total_Q':'q', 'E_Total_Q':'E_Total_Q', 'Total_U':'u', 'E_Total_U':'E_Total_U', 'Total_V':'v', 'E_Total_V':'E_Total_V', 'Linear_Pol_frac':'Linear_Pol_frac', 'Elow_Linear_Pol_frac':'Elow_Linear_Pol_frac', 'Ehigh_Linear_Pol_frac':'Ehigh_Linear_Pol_frac', 'Circ_Pol_Frac':'Circ_Pol_Frac', 'Elow_Circ_Pol_Frac':'Elow_Circ_Pol_Frac', 'Ehigh_Circ_Pol_Frac':'Ehigh_Circ_Pol_Frac', 'Total_Pol_Frac':'Total_Pol_Frac', 'Elow_Total_Pol_Frac':'Elow_Total_Pol_Frac', 'Ehigh_Total_Pol_Frac':'Ehigh_Total_Pol_Frac', 'Linear_Pol_Ang':'Linear_Pol_Ang', 'E_Linear_Pol_Ang':'E_Linear_Pol_Ang'}
#Dictionary for classifying a parameter as a general parameter or a polarization-specific parameter
dict_pol_flag = {'Gaus_id':0, 'Isl_id':0, 'Source_id':0, 'Wave_id':0, 'RA':0, 'E_RA':0, 'DEC':0, 'E_DEC':0, 'Total_flux':0, 'E_Total_flux':0, 'Peak_flux':0, 'E_Peak_flux':0, 'Xposn':0, 'E_Xposn':0, 'Yposn':0, 'E_Yposn':0, 'Maj':0, 'E_Maj':0, 'Min':0, 'E_Min':0, 'PA':0, 'E_PA':0, 'Maj_img_plane':0, 'E_Maj_img_plane':0, 'Min_img_plane':0, 'E_Min_img_plane':0, 'PA_img_plane':0, 'E_PA_img_plane':0, 'DC_Maj':0, 'E_DC_Maj':0, 'DC_Min':0, 'E_DC_Min':0, 'DC_PA':0, 'E_DC_PA':0, 'DC_Maj_img_plane':0, 'E_DC_Maj_img_plane':0, 'DC_Min_img_plane':0, 'E_DC_Min_img_plane':0, 'DC_PA_img_plane':0, 'E_DC_PA_img_plane':0, 'Isl_Total_flux':0, 'E_Isl_Total_flux':0, 'Isl_rms':0, 'Isl_mean':0, 'Resid_Isl_rms':0, 'Resid_Isl_mean':0, 'S_Code':0, 'Total_Q':1, 'E_Total_Q':1, 'Total_U':1, 'E_Total_U':1, 'Total_V':1, 'E_Total_V':1, 'Linear_Pol_frac':1, 'Elow_Linear_Pol_frac':1, 'Ehigh_Linear_Pol_frac':1, 'Circ_Pol_Frac':1, 'Elow_Circ_Pol_Frac':1, 'Ehigh_Circ_Pol_Frac':1, 'Total_Pol_Frac':1, 'Elow_Total_Pol_Frac':1, 'Ehigh_Total_Pol_Frac':1, 'Linear_Pol_Ang':1, 'E_Linear_Pol_Ang':1}
lines = [line.strip() for line in open(gaul)]
for line in range(len(lines)):
if lines[line]:
if lines[line].split()[0] is not '#':
gaul_params = lines[line-1].split()[1:] #Parameter list is last line in gaul file that begins with a '#'
break
# Initialize lists for general and polarization parameters
lsm_params_general = []
lsm_params_polarization = []
for param in gaul_params:
if dict_pol_flag[param] is 0:
lsm_params_general.append(dict_gaul2lsm[param])
if dict_pol_flag[param] is 1:
lsm_params_polarization.append(dict_gaul2lsm[param])
general_params_string = ' '.join(lsm_params_general)
pol_params_string = " " + ' '.join(lsm_params_polarization)
tigger_convert(gaul,output,"-t","ASCII","--format", general_params_string + (pol_params_string if pol else ""),
"-f","--rename",
"--cluster-dist",cluster,
"--min-extent",MIN_EXTENT,
split_args=False,
*args);