def do_sourcefinding(imagename, si=True):
# get beam info manually. SKA image seems to cause PyBDSF issues finding this info.
f = fits.open(imagename)
beam_maj = f[0].header['BMAJ']
beam_min = f[0].header['BMIN']
#beam_pa = f[0].header['BPA'] # not in SKA fits header, but we know it's circular
beam_pa = 0
f.close()
# using some sensible and thorough hyper-parameters. PSF_vary and adaptive_rms_box is more computationally intensive, but needed.
if si == True:
img = bdsf.process_image(imagename, adaptive_rms_box=True, spectralindex_do=True, advanced_opts=True,\
atrous_do=False, psf_vary_do=True, psf_snrcut=5.0, psf_snrcutstack=10.0,\
output_opts=True, output_all=True, opdir_overwrite='append', beam=(beam_maj, beam_min, beam_pa),\
blank_limit=None, thresh='hard', thresh_isl=5.0, thresh_pix=7.0, psf_snrtop=0.30,\
collapse_mode='single') # use 560 Mhz image as ch0
# save the img object as a pickle file, so we can do interactive checks after pybdsf has run
save_obj(img, 'pybdsf_processimage_' + imagename[:-5])
if si == False:
img = bdsf.process_image(imagename, adaptive_rms_box=True, advanced_opts=True,\
atrous_do=False, psf_vary_do=True, psf_snrcut=5.0, psf_snrcutstack=10.0,\
output_opts=True, output_all=True, opdir_overwrite='append', beam=(beam_maj, beam_min, beam_pa),\
blank_limit=None, thresh='hard', thresh_isl=5.0, thresh_pix=7.0, psf_snrtop=0.30)
# save the img object as a pickle file, so we can do interactive checks after pybdsf has run
save_obj(img, 'pybdsf_processimage_noSI_' + imagename[:-5])
def find_sources(self):
"""Run PyBDSF ``process_image()`` task using object
attributes as parameter inputs. Returns ``None`` if
PyBDSF fails. Wrapped in a timeout function so
processing is killed if taking longer than 5 minutes.
"""
sf_logger.info('Extracting sources...')
start = datetime.now()
opts = self.get_attr()
with warnings.catch_warnings():
warnings.filterwarnings('ignore', r'invalid value')
try:
out = bdsf.process_image(opts)
except:
out = None
try:
sf_logger.info(' -- found {} sources in {:.2f} seconds'.format(
out.nsrc, (datetime.now() - start).total_seconds()))
except AttributeError:
try:
sf_logger.info(' -- found {} islands in {:.2f} seconds'.format(
out.nisl, (datetime.now() - start).total_seconds()))
except AttributeError:
sf_logger.info(' -- PyBDSF failed to process image.')
return out
def RunBDSM(self,
filename,
rmsmean_map_filename=None,
WriteNoise=True,
Ratio=None):
CatName = filename[:-5] + ".pybdsm.gaul.fits"
if os.path.isfile(CatName):
print >> log, ModColor.Str("File %s exists" % CatName, col="green")
print >> log, ModColor.Str(" Skipping...", col="green")
return
rmsmean_map_filename = []
if self.rmsmean_map:
l = self.rmsmean_map
l = l.replace("[", "")
l = l.replace("]", "")
l = l.split(",")
rmsmean_map_filename = l
img = bdsf.process_image(filename,
thresh_isl=self.bdsm_thresh_isl,
thresh_pix=self.bdsm_thresh_pix,
rms_box=self.bdsm_rms_box,
rmsmean_map_filename=rmsmean_map_filename)
img.write_catalog(catalog_type="srl", clobber=True, format="fits")
img.write_catalog(catalog_type="gaul", clobber=True, format="fits")
if WriteNoise:
img.export_image(img_type="rms", clobber=True)
img.export_image(img_type="mean", clobber=True)
def make_mask(image_name, mask_name=None, threshisl=5, atrous_do=False, rmsbox=(100,30), mask_combine=None):
import sys, os
import numpy as np
from astropy.io import fits as pyfits
import bdsf
# wavelets are required to fit gaussians
if atrous_do: stop_at = None
else: stop_at = 'isl'
# DO THE SOURCE DETECTION
img = bdsf.process_image(image_name, rms_box=rmsbox, frequency=54e6, \
thresh_isl=float(threshisl), thresh_pix=float(threshisl*3./5), atrous_do=atrous_do, atrous_jmax=3, \
adaptive_rms_box=True, adaptive_thresh=100, rms_box_bright=(30,10), stop_at=stop_at, quiet=True, debug=False)
# WRITE THE MASK FITS
if mask_name == None: mask_name = image_name+'.newmask'
if os.path.exists(mask_name): os.system('rm -r ' + mask_name)
img.export_image(img_type='island_mask', img_format='fits', outfile=mask_name)
del img
# do an pixel-by-pixel "OR" operation with a given mask
if not mask_combine is None:
print("Doing a pix-by-pix OR with %s." % mask_combine)
with pyfits.open(mask_combine) as fits:
data_comb = fits[0].data
with pyfits.open(mask_name) as fits:
data = fits[0].data
assert data.shape() == data_comb.shape()
data[(data_comb == 1.)] = 1.
fits[0].data = data
fits.writeto(mask_name, overwrite=True)
return mask_name
def find_sources(self):
"""
Find sources in smoothed & pb-corrected image
Write them to a temp file in working directory
"""
#dont' do source finding if output already exists
if not os.path.exists(self.bdsf_output) and not self.skipcheck:
try:
#use all defaults ot match what ALexander does
bdsf.process_image(self.pbsmfits).write_catalog(
outfile=self.bdsf_output, format='ascii', clobber=True)
except Exception as e:
self.status = False
print(
("Source finding failed for "
"beam {0} of taskid {1}").format(self.beam, self.taskid))
print(e)
def extract_tier1(input_image, options=options_tier1):
"""
Extract the sources from a fits image using PyBDSF.
By default it applies the options used for Tier1.
"""
img = bdsf.process_image(input_image, **options)
if not options["output_all"]:
img.write_catalog(format='bbs', clobber=True)
img.write_catalog(format='fits', catalog_type='gaul', clobber=True)
img.write_catalog(format='fits', catalog_type='srl', clobber=True)
def find_sources(self, sf="aegean", options={}):
outfile = self.imagename.replace(".fits", "_comp.csv")
if sf == "aegean":
command = "aegean "
for opt in sorted(options):
if opt == "table":
continue
command += "--{} {} ".format(opt, options[opt])
command += "--table {} ".format(
self.imagename.replace(".fits", ".csv"))
command += self.image + " "
command += "> " + self.imagename.replace(".fits", "_aegean.log")
self.logger.debug("Aegean command: {}".format(command))
subprocess.call(command, shell=True)
elif sf == "pybdsf":
self.logger.info("Running source finding with PyBDSF...")
kwargs = options
# procimg_kwargs.update(options)
pybdsf_img = bdsf.process_image(self.image, **kwargs)
pybdsf_img.write_catalog(outfile=outfile + ".original",
catalog_type="gaul",
format="csv")
utils.pybdsf2aegean(outfile + ".original", outfile)
self.logger.info(
"PyBDSF output converted to aegean format and placed in {}.".
format(outfile))
elif sf == "selavy":
self.logger.info("Running source finding with Selavy...")
selavy_parset_name = self.imagename.replace(".fits", ".selavy.in")
#Add the image to the parset
selavy_outputname = self.imagename.replace(".fits", ".txt")
options["Selavy.image"] = self.image
options["Selavy.resultsFile"] = selavy_outputname
self._write_selavy_parset(self, selavy_parset_name, options)
self.logger.info(
"Written Selavy parset file {}.".format(selavy_parset_name))
if "Selavy.nsubx" in options:
nsubx = int(options["Selavy.nsubx"])
if "Selavy.nsuby" in options:
nsuby = int(options["Selavy.nsuby"])
num_procs = nsubx * nsuby + 1
command = "mpirun -n {} selavy -c {} > ".format(
num_procs, selavy_parset_name) + self.imagename.replace(
".fits", "_selavy.log")
subprocess.call(command, shell=True)
utils.selavy2aegean(
selavy_outputname.replace(".txt", ".components.txt"), outfile)
self.logger.info(
"Selavy output converted to aegean format and placed in {}.".
format(outfile))
def make_mask(image_name,
mask_name=None,
threshisl=5,
atrous_do=False,
rmsbox=(100, 10),
adaptive_thresh=50,
mask_combine=None,
write_srl=False):
import sys, os
import numpy as np
from astropy.io import fits as pyfits
import bdsf
# wavelets are required to fit gaussians
if atrous_do or write_srl: stop_at = None
else: stop_at = 'isl'
# DO THE SOURCE DETECTION
img = bdsf.process_image(image_name, rms_box=rmsbox, frequency=54e6, \
thresh_isl=float(threshisl), thresh_pix=float(threshisl*3/5.), rms_map=True, mean_map='zero', atrous_do=atrous_do, atrous_jmax=4, \
adaptive_rms_box=True, adaptive_thresh=adaptive_thresh, rms_box_bright=(30,5), \
flagging_opts=True, flag_maxsize_fwhm=0.5, stop_at=stop_at, quiet=True, debug=False)
# WRITE THE MASK FITS
if mask_name == None: mask_name = image_name + '.newmask'
if os.path.exists(mask_name): os.system('rm -r ' + mask_name)
img.export_image(img_type='island_mask',
img_format='fits',
outfile=mask_name,
clobber=True)
# WRITE CATALOGUE
if write_srl:
img.write_catalog(format='bbs',
catalog_type='gaul',
outfile=mask_name.replace('fits', 'skymodel'),
clobber=True)
del img
# do an pixel-by-pixel "OR" operation with a given mask
if not mask_combine is None:
print("Doing a pix-by-pix OR with %s." % mask_combine)
with pyfits.open(mask_combine) as fits:
data_comb = fits[0].data
with pyfits.open(mask_name) as fits:
data = fits[0].data
assert data.shape() == data_comb.shape()
data[(data_comb == 1.)] = 1.
fits[0].data = data
fits.writeto(mask_name, overwrite=True)
return mask_name
def do_sourcefinding(imagename):
# get beam info manually. SKA image seems to cause PyBDSF issues finding this info.
f = fits.open(imagename)
beam_maj = f[0].header['BMAJ']
beam_min = f[0].header['BMIN']
#beam_pa = f[0].header['BPA'] # not in SKA fits header, but we know it's circular
beam_pa = 0
f.close()
# Run sourcefinding using some sensible hyper-parameters. PSF_vary and adaptive_rms_box is more computationally intensive, off for now
img = bdsf.process_image(imagename, adaptive_rms_box=False, advanced_opts=True,\
atrous_do=False, psf_vary_do=False, psf_snrcut=5.0, psf_snrcutstack=10.0,\
output_opts=True, output_all=True, opdir_overwrite='append', beam=(beam_maj, beam_min, beam_pa),\
blank_limit=None, thresh='hard', thresh_isl=5.0, thresh_pix=7.0, psf_snrtop=0.30) #
def create_mask(self, image, mask, threshold, theoretical_noise, beampars=None, rms_map=None):
"""
Creates a mask from an image using pybdsf
image (string): Input image to use in MIRIAD format
mask (string): Output mask image in MIRIAD format
threshold (float): Threshold in Jy to use
theoretical_noise (float): Theoretical noise for calculating the adaptive threshold parameter inside pybdsf
"""
convim.mirtofits(image, image + '.fits')
bdsf_threshold = threshold / theoretical_noise
if beampars:
# bdsf.process_image(image + '.fits', stop_at='isl', thresh_isl=bdsf_threshold, beam=beampars, adaptive_rms_box=True, rms_map=rms_map).export_image(outfile=mask + '.fits', img_format='fits', img_type='island_mask', pad_image=True)
bdsf.process_image(image + '.fits', stop_at='isl', thresh_isl=bdsf_threshold, beam=beampars, adaptive_rms_box=True, rms_map=False, rms_value=theoretical_noise).export_image(outfile=mask + '.fits', img_format='fits', img_type='island_mask', pad_image=True)
else:
bdsf.process_image(image + '.fits', stop_at='isl', thresh_isl=bdsf_threshold, adaptive_rms_box=True, rms_map=False, rms_value=theoretical_noise).export_image(outfile=mask + '.fits', img_format='fits', img_type='island_mask', pad_image=True)
if os.path.isfile(mask + '.fits'):
# Add a random number to the masks to make it viewable in kvis
fitsmask = pyfits.open(mask + '.fits')
fitsmask_data = fitsmask[0].data
fitsmask_hdr = fitsmask[0].header
rand_array = np.random.rand(int(fitsmask_hdr['NAXIS1']), int(fitsmask_hdr['NAXIS2']))
fitsmask[0].data = np.multiply(rand_array, fitsmask_data)
fitsmask.writeto(mask + '.fits', clobber=True)
# Convert mask to MIRIAD and generate a usable one for MIRIAD
convim.fitstomir(mask + '.fits', mask + '_pybdsf')
maths = lib.miriad('maths')
maths.out = mask
maths.exp = '"<' + mask + '_pybdsf>"'
maths.mask = '"<' + mask + '_pybdsf>.gt.0' + '"'
maths.go()
managefiles.director(self, 'rm', image + '.fits.pybdsf.log')
managefiles.director(self, 'rm', image + '.fits')
managefiles.director(self, 'rm', mask + '.fits')
managefiles.director(self, 'rm', mask + '_pybdsf')
else:
pass
def calc_shift(self, ref_cat, separation=15):
"""
Find a shift cross-matching source extracted from the image and a given catalog
separation in arcsec
"""
import bdsf
from astropy.coordinates import match_coordinates_sky
from astropy.coordinates import SkyCoord
import astropy.units as u
img_cat = self.imagefile + '.cat'
if not os.path.exists(img_cat):
bdsf_img = bdsf.process_image(self.imagefile, rms_box=(100,30), \
thresh_pix=5, thresh_isl=3, atrous_do=False, \
adaptive_rms_box=True, adaptive_thresh=100, rms_box_bright=(30,10))
bdsf_img.write_catalog(outfile=img_cat,
catalog_type='srl',
format='fits',
clobber=True)
# TODO: add iterations to remove off sources until convergence
# read catlogue
ref_t = Table.read(ref_cat)
img_t = Table.read(img_cat)
# cross match
idx_match, sep, _ = match_coordinates_sky(SkyCoord(ref_t['RA'], ref_t['DEC']),\
SkyCoord(img_t['RA'], img_t['DEC']))
idx_match_img = idx_match[sep < separation * u.arcsec]
idx_match_ref = np.arange(0, len(ref_t))[sep < separation * u.arcsec]
# find & apply shift
if len(idx_match) == 0:
logging.warning('No match found in TGSS.')
return
dra = ref_t['RA'][idx_match_ref] - img_t['RA'][idx_match_img]
dra[dra > 180] -= 360
dra[dra < -180] += 360
ddec = ref_t['DEC'][idx_match_ref] - img_t['DEC'][idx_match_img]
self.apply_shift(np.mean(dra), np.mean(ddec))
# clean up
if not args.save:
os.system('rm ' + img_cat)
def make_catalogue(self):
"""
Create catalogue for this image
"""
import bdsf
from astropy.table import Table
img_cat = self.imagefile+'.cat'
if not os.path.exists(img_cat):
bdsf_img = bdsf.process_image(self.imagefile, rms_box=(100,30), \
thresh_pix=5, thresh_isl=3, atrous_do=False, \
adaptive_rms_box=True, adaptive_thresh=100, rms_box_bright=(30,10), quiet=True)
bdsf_img.write_catalog(outfile=img_cat, catalog_type='srl', format='fits', clobber=True)
else:
logging.warning('%s already exists, using it.' % img_cat)
self.cat = Table.read(img_cat)
logging.debug('%s: Number of sources detected: %i' % (self.imagefile, len(self.cat)) )
def make_catalogue(self):
"""
Create catalogue for this image
"""
import bdsf
from astropy.table import Table
img_cat = self.imagefile+'.cat'
if not os.path.exists(img_cat):
bdsf_img = bdsf.process_image(self.imagefile, rms_box=(100,30), \
thresh_pix=5, thresh_isl=3, atrous_do=False, \
adaptive_rms_box=True, adaptive_thresh=100, rms_box_bright=(30,10), quiet=True)
bdsf_img.write_catalog(outfile=img_cat, catalog_type='srl', format='fits', clobber=True)
else:
logging.warning('%s already exists, using it.' % img_cat)
self.cat = Table.read(img_cat)
logging.debug('%s: Number of sources detected: %i' % (self.imagefile, len(self.cat)) )
def make_catalogue(self):
"""
Create catalogue for this image.
"""
import bdsf
from astropy.table import Table
img_cat = self.imagefile + '.cat'
if not os.path.exists(img_cat):
bdsf_img = bdsf.process_image(self.imagefile, rms_box=(100,30), \
thresh_pix=5, thresh_isl=3, atrous_do=False, \
adaptive_rms_box=True, adaptive_thresh=100, rms_box_bright=(30,10), quiet=True)
bdsf_img.write_catalog(outfile=img_cat,
catalog_type='srl',
format='fits',
clobber=True)
bdsf_img.write_catalog(outfile=img_cat.replace(
'.cat', '.skymodel'),
catalog_type='gaul',
format='bbs',
bbs_patches='source',
clobber=True,
srcroot='src')
else:
logging.warning('%s already exists, using it.' % img_cat)
cat = Table.read(img_cat)
# remove extended sources
extended_src = (cat['Peak_flux'] /
cat['Total_flux']) < 0.1 # ~extended source
extended_src[cat['S_Code'] == 'M'] = True # multiple-gaussian source
extended_src[cat['S_Code'] ==
'C'] = True # one gaussian + other sources island
# remove same sources from skymodel
cat_lsm = lsm.load(img_cat.replace('.cat', '.skymodel'))
for srcid in cat[extended_src]['Source_id']:
cat_lsm.remove(f'Patch == src_patch_s{srcid}')
cat.remove_rows(np.argwhere(extended_src))
self.cat = cat
self.cat_lsm = cat_lsm
logging.debug(
'%s: Number of sources detected: %i; removed %i extended sources.'
% (self.imagefile, len(self.cat), sum(extended_src)))
def makeLSM(infits): #,mybeam,myfreq):
img = bdsf.process_image(infits, thresh_pix=9.0,
thresh_isl=8.0) #,beam=mybeam,frequency=myfreq)
foundsrcs = img.write_catalog(format='ascii',
catalog_type='gaul',
clobber=True,
incl_empty=True)
gaul = infits.replace('.fits', '.pybdsm.gaul')
oplsm = gaul.replace('gaul', 'lsm.html')
if foundsrcs:
tiggerConvert(gaul)
if cleanup:
os.system('rm ' + gaul)
os.system('rm ' + gaul.replace('.pybdsm.gaul', '.fits.pybdsm.log'))
gi('Wrote LSM: ' + oplsm)
return (oplsm, True)
else:
ri('No sources found in ' + str(infits))
ri('Could be corrupt image, could be trouble source entering null')
return (oplsm, False)
def _process_image(self, fitsfile, thresh_isl, thresh_pix):
"""
Executes PyBDSF on the fitsfile
Parameters
----------
fitsfile : string
Name of input fitsfile
thresh_isl : float
Threshold for the island boundary in number of sigma above the mean.
Determines extent of island used for fitting.
thresh_pix : float
Source detection threshold: threshold for the island peak in number
of sigma above the mean.
"""
bdsf_dict = {
'rms_map': None,
'thresh': 'hard',
'thresh_isl': thresh_isl,
'thresh_pix': thresh_pix
}
return bdsf.process_image(fitsfile, **bdsf_dict)
def run_pybdsf(fitsname, detectimage, outcat='skymodel_1asec_lbregion_pybdsf'):
''' Run PyBDSF on an image, using standard SKSP settings.
Two catalogues are written when finished:
- A CSV catalogue with all columns present.
- A BBS formatted catalogue, suitable for e.g. DPPP.
Args:
fitsname (str): path to the image from which to extract the fluxes.
detectimage (str): path to the image on which to run source detection.
Returns:
None
'''
# Pull the reference frequency from the header.
f = fits.open(fitsname)
restfrq = f[0].header['CRVAL3']
# Run PyBDSF with standard SKSP settings.
res = bdsf.process_image(fitsname, detection_image=detectimage, thresh_isl=4.0, thresh_pix=5.0, rms_box=(150, 15), rms_map=True, mean_map='zero', ini_method='intensity', adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(60, 15), group_by_isl=False, group_tol=10.0, output_opts=True, output_all=True, atrous_do=True, atrous_jmax=4, flagging_opts=True, flag_maxsize_fwhm=0.5, advanced_opts=True, blank_limit=None, frequency=restfrq)
# Write out a catalog.
res.write_catalog(outfile=outcat + '.csv', bbs_patches='source', catalog_type='gaul', format='csv')
res.write_catalog(outfile=outcat + '.bbs', bbs_patches='source', catalog_type='gaul', format='bbs')
def calc_shift(self, ref_cat, separation=15):
"""
Find a shift cross-matching source extracted from the image and a given catalog
separation in arcsec
"""
import bdsf
from astropy.coordinates import match_coordinates_sky
from astropy.coordinates import SkyCoord
import astropy.units as u
img_cat = self.imagefile+'.cat'
if not os.path.exists(img_cat):
bdsf_img = bdsf.process_image(self.imagefile, rms_box=(100,30), \
thresh_pix=5, thresh_isl=3, atrous_do=False, \
adaptive_rms_box=True, adaptive_thresh=100, rms_box_bright=(30,10), quiet=True)
bdsf_img.write_catalog(outfile=img_cat, catalog_type='srl', format='fits', clobber=True)
# read catlogue
ref_t = Table.read(ref_cat)
img_t = Table.read(img_cat)
# cross match
idx_match, sep, _ = match_coordinates_sky(SkyCoord(ref_t['RA'], ref_t['DEC']),\
SkyCoord(img_t['RA'], img_t['DEC']))
idx_match_img = idx_match[sep<separation*u.arcsec]
idx_match_ref = np.arange(0,len(ref_t))[sep<separation*u.arcsec]
# find & apply shift
if len(idx_match) == 0:
logging.warning('No match found in TGSS.')
return
dra = ref_t['RA'][idx_match_ref] - img_t['RA'][idx_match_img]
dra[ dra>180 ] -= 360
dra[ dra<-180 ] += 360
ddec = ref_t['DEC'][idx_match_ref] - img_t['DEC'][idx_match_img]
self.apply_shift(np.mean(dra), np.mean(ddec))
# clean up
if not args.save:
os.system('rm '+img_cat)
def sfind_image(catprefix,pbimage,nonpbimage,sfind_pixel_fraction,options=None):
if options is None:
options = o
f = fits.open(nonpbimage)
imsizex = f[0].header['NAXIS1']
imsizey = f[0].header['NAXIS2']
f.close()
kwargs={}
if options['sfind_pixel_fraction']<1.0:
lowerx,upperx = int(((1.0-sfind_pixel_fraction)/2.0)*imsizex),int(((1.0-sfind_pixel_fraction)/2.0)*imsizex + sfind_pixel_fraction*imsizex)
lowery,uppery = int(((1.0-sfind_pixel_fraction)/2.0)*imsizey),int(((1.0-sfind_pixel_fraction)/2.0)*imsizey + sfind_pixel_fraction*imsizey)
kwargs['trim_box']=(lowerx,upperx,lowery,uppery)
if options['restart'] and os.path.isfile(catprefix +'.cat.fits'):
warn('File ' + catprefix +'.cat.fits already exists, skipping source finding step')
else:
img = bdsm.process_image(pbimage, detection_image=nonpbimage, thresh_isl=4.0, thresh_pix=5.0, rms_box=(160,50), rms_map=True, mean_map='zero', ini_method='intensity', adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(60,15), group_by_isl=False, group_tol=10.0,output_opts=True, output_all=True, atrous_do=True,atrous_jmax=4, flagging_opts=True, flag_maxsize_fwhm=0.5,advanced_opts=True, ncores=options['NCPU'], blank_limit=None,**kwargs)
img.write_catalog(outfile=catprefix +'.cat.fits',catalog_type='srl',format='fits',correct_proj='True')
img.export_image(outfile=catprefix +'.rms.fits',img_type='rms',img_format='fits',clobber=True)
img.export_image(outfile=catprefix +'.resid.fits',img_type='gaus_resid',img_format='fits',clobber=True)
img.export_image(outfile=catprefix +'.pybdsmmask.fits',img_type='island_mask',img_format='fits',clobber=True)
img.write_catalog(outfile=catprefix +'.cat.reg',catalog_type='srl',format='ds9',correct_proj='True')
def sfind_image(catprefix,pbimage,nonpbimage,sfind_pixel_fraction,options=None):
if options is None:
options = o
f = fits.open(nonpbimage)
imsizex = f[0].header['NAXIS1']
imsizey = f[0].header['NAXIS2']
f.close()
kwargs={}
if o['sfind_pixel_fraction']<1.0:
lowerx,upperx = int(((1.0-sfind_pixel_fraction)/2.0)*imsizex),int(((1.0-sfind_pixel_fraction)/2.0)*imsizex + sfind_pixel_fraction*imsizex)
lowery,uppery = int(((1.0-sfind_pixel_fraction)/2.0)*imsizey),int(((1.0-sfind_pixel_fraction)/2.0)*imsizey + sfind_pixel_fraction*imsizey)
kwargs['trim_box']=(lowerx,upperx,lowery,uppery)
if options['restart'] and os.path.isfile(catprefix +'.cat.fits'):
warn('File ' + catprefix +'.cat.fits already exists, skipping source finding step')
else:
img = bdsm.process_image(pbimage, detection_image=nonpbimage, thresh_isl=4.0, thresh_pix=5.0, rms_box=(160,50), rms_map=True, mean_map='zero', ini_method='intensity', adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(60,15), group_by_isl=False, group_tol=10.0,output_opts=True, output_all=True, atrous_do=True,atrous_jmax=4, flagging_opts=True, flag_maxsize_fwhm=0.5,advanced_opts=True, blank_limit=None,**kwargs)
img.write_catalog(outfile=catprefix +'.cat.fits',catalog_type='srl',format='fits',correct_proj='True')
img.export_image(outfile=catprefix +'.rms.fits',img_type='rms',img_format='fits',clobber=True)
img.export_image(outfile=catprefix +'.resid.fits',img_type='gaus_resid',img_format='fits',clobber=True)
img.export_image(outfile=catprefix +'.pybdsmmask.fits',img_type='island_mask',img_format='fits',clobber=True)
img.write_catalog(outfile=catprefix +'.cat.reg',catalog_type='srl',format='ds9',correct_proj='True')
def make_mask(image_name, mask_name=None, threshisl=5, atrous_do=False, rmsbox=(100,30), mask_combine=None):
import sys, os
import numpy as np
from astropy.io import fits as pyfits
import bdsf
# wavelets are required to fit gaussians
if atrous_do: stop_at = None
else: stop_at = 'isl'
# DO THE SOURCE DETECTION
img = bdsf.process_image(image_name, rms_box=rmsbox, \
thresh_isl=int(threshisl), atrous_do=atrous_do, atrous_jmax=3, \
adaptive_rms_box=True, adaptive_thresh=100, rms_box_bright=(30,10), stop_at=stop_at, quiet=True, debug=False)
# WRITE THE MASK FITS
if mask_name == None: mask_name = image_name+'.newmask'
if os.path.exists(mask_name): os.system('rm -r ' + mask_name)
print('Making mask:', mask_name)
img.export_image(img_type='island_mask', img_format='fits', outfile=mask_name)
del img
# do an pixel-by-pixel "OR" operation with a given mask
if not mask_combine is None:
print("Doing a pix-by-pix OR with %s." % mask_combine)
with pyfits.open(mask_combine) as fits:
data_comb = fits[0].data
with pyfits.open(mask_name) as fits:
data = fits[0].data
assert data.shape() == data_comb.shape()
data[(data_comb == 1.)] = 1.
fits[0].data = data
fits.writeto(mask_name, clobber=True)
return mask_name
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jul 14 08:04:50 2020
@author: kabelo
"""
import bdsf
import glob
input_images = []
for image in glob.glob("*.fits"):
input_images.append(image)
for input_image in input_images:
img = bdsf.process_image(input_image, rms_box=(20, 10))
img.write_catalog('./AbellOutput/', format='fits', catalog_type='srl')
img.export_image('./AbellOutput/', img_type='gaus_resid')
img.export_image('./AbellOutput/',
img_type='gaus_model',
outfile=input_image + '.model')
infile = 'mosaic-blanked.fits'
if args.no_highres:
catprefix = 'low-mosaic'
infile = 'low-mosaic-blanked.fits'
img = bdsm.process_image(infile,
thresh_isl=4.0,
thresh_pix=5.0,
rms_box=(150, 15),
rms_map=True,
mean_map='zero',
ini_method='intensity',
adaptive_rms_box=True,
adaptive_thresh=150,
rms_box_bright=(60, 15),
group_by_isl=False,
group_tol=10.0,
output_opts=True,
output_all=True,
atrous_do=True,
atrous_jmax=4,
flagging_opts=True,
flag_maxsize_fwhm=0.5,
advanced_opts=True,
blank_limit=None,
frequency=restfrq)
img.write_catalog(outfile=catprefix + '.cat.fits',
catalog_type='srl',
format='fits',
correct_proj='True')
img.export_image(outfile=catprefix + '.rms.fits',
def do_offsets(o):
# o is the options file
if o['mode']!='normal' and o['mode']!='test':
raise NotImplementedError('Offsets called with mode '+o['mode'])
image_root='image_full_ampphase_di_m.NS'
method=o['method']
report('Determining astrometric offsets with method '+method+' in mode '+o['mode'])
report('Merging downloaded catalogues')
if os.path.isfile(method+'.fits'):
warn('Merged file exists, reading from disk instead')
data=Table.read(method+'.fits')
else:
if method=='pslocal':
data=Table.read(method+'/'+method+'.txt',format='ascii')
data['RA'].name='ra'
data['DEC'].name='dec'
data.write(method+'.fits')
else:
kwargs={}
if 'panstarrs' in method:
kwargs['rastr']='ramean'
kwargs['decstr']='decmean'
data=merge_cat(method,**kwargs)
if o['mode']=='test':
image_root+='_shift'
method+='-test'
report('Running PyBDSM on LOFAR image, please wait...')
catfile=image_root+'.offset_cat.fits'
gaulfile=catfile.replace('cat','gaul')
if os.path.isfile(catfile):
warn('Catalogue already exists, skipping pybdsf run')
else:
if o['mode']=='test':
suffix='facetRestored'
else:
suffix='restored'
pbimage=image_root+'.int.'+suffix+'.fits'
nonpbimage=image_root+'.app.'+suffix+'.fits'
img = bdsm.process_image(pbimage, detection_image=nonpbimage, thresh_isl=4.0, thresh_pix=5.0, rms_box=(150,15), rms_map=True, mean_map='zero', ini_method='intensity', adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(60,15), group_by_isl=False, group_tol=10.0,output_opts=True, output_all=True, atrous_do=False, flagging_opts=True, flag_maxsize_fwhm=0.5,advanced_opts=True, blank_limit=None)
img.write_catalog(outfile=catfile,catalog_type='srl',format='fits',correct_proj='True')
img.write_catalog(outfile=gaulfile,catalog_type='gaul',format='fits',correct_proj='True')
lofar=Table.read(catfile)
print len(lofar),'LOFAR sources before filtering'
filter=(lofar['E_RA']*3600.0)<2.0
filter&=(lofar['E_DEC']*3600.0)<2.0
filter&=(lofar['Maj']*3600.0)<10
lofar=lofar[filter]
print len(lofar),'LOFAR sources after filtering'
regfile=image_root+'.tessel.reg'
cra,cdec=get_centpos()
report('Set up structure')
NDir=np.load("image_dirin_SSD_m.npy.ClusterCat.npy").shape[0]
oo=Offsets(method,n=NDir,imroot=image_root,cellsize=o['cellsize'],fitmethod=o['fit'])
report('Label table')
lofar_l=oo.r.add_facet_labels(lofar)
report('Finding offsets')
oo.find_offsets(lofar_l,data)
report('Fitting offsets')
oo.fit_offsets()
report('Making plots and saving output')
#oo.plot_fits(method+'-fits.pdf')
oo.save_fits()
oo.plot_offsets()
if 'test' not in o['mode']:
oo.save(method+'-fit_state.pickle')
report('Making astrometry error map, please wait')
oo.make_astrometry_map('astromap.fits',20)
oo.offsets_to_facetshift('facet-offset.txt')
def main(image_name,
mask_name,
atrous_do=False,
threshisl=0.0,
threshpix=0.0,
rmsbox=None,
rmsbox_bright=(35, 7),
iterate_threshold=False,
adaptive_rmsbox=False,
img_format='fits',
threshold_format='float',
trim_by=0.0,
vertices_file=None,
atrous_jmax=6,
pad_to_size=None,
skip_source_detection=False,
region_file=None,
nsig=1.0,
reference_ra_deg=None,
reference_dec_deg=None,
cellsize_deg=0.000417,
use_adaptive_threshold=False,
adaptive_thresh=150.0):
"""
Make a clean mask and return clean threshold
Parameters
----------
image_name : str
Filename of input image from which mask will be made. If the image does
not exist, a template image with center at (reference_ra_deg,
reference_dec_deg) will be made internally
mask_name : str
Filename of output mask image
atrous_do : bool, optional
Use wavelet module of PyBDSF?
threshisl : float, optional
Value of thresh_isl PyBDSF parameter
threshpix : float, optional
Value of thresh_pix PyBDSF parameter
rmsbox : tuple of floats, optional
Value of rms_box PyBDSF parameter
rmsbox_bright : tuple of floats, optional
Value of rms_box_bright PyBDSF parameter
iterate_threshold : bool, optional
If True, threshold will be lower in 20% steps until
at least one island is found
adaptive_rmsbox : tuple of floats, optional
Value of adaptive_rms_box PyBDSF parameter
img_format : str, optional
Format of output mask image (one of 'fits' or 'casa')
threshold_format : str, optional
Format of output threshold (one of 'float' or 'str_with_units')
trim_by : float, optional
Fraction by which the perimeter of the output mask will be
trimmed (zeroed)
vertices_file : str, optional
Filename of file with vertices (must be a pickle file containing
a dictionary with the vertices in the 'vertices' entry)
atrous_jmax : int, optional
Value of atrous_jmax PyBDSF parameter
pad_to_size : int, optional
Pad output mask image to a size of pad_to_size x pad_to_size
skip_source_detection : bool, optional
If True, source detection is not run on the input image
region_file : str, optional
Filename of region file in CASA format. If given, no mask image
is made (the region file is used as the clean mask)
nsig : float, optional
Number of sigma of returned threshold value
reference_ra_deg : float, optional
RA for center of output mask image
reference_dec_deg : float, optional
Dec for center of output mask image
cellsize_deg : float, optional
Size of a pixel in degrees
use_adaptive_threshold : bool, optional
If True, use an adaptive threshold estimated from the negative values in
the image
adaptive_thresh : float, optional
If adaptive_rmsbox is True, this value sets the threshold above
which a source will use the small rms box
Returns
-------
result : dict
Dict with nsig-sigma rms threshold
"""
if rmsbox is not None and type(rmsbox) is str:
rmsbox = eval(rmsbox)
if type(rmsbox_bright) is str:
rmsbox_bright = eval(rmsbox_bright)
if pad_to_size is not None and type(pad_to_size) is str:
pad_to_size = int(pad_to_size)
if type(atrous_do) is str:
if atrous_do.lower() == 'true':
atrous_do = True
threshisl = 4.0 # override user setting to ensure proper source fitting
else:
atrous_do = False
if type(iterate_threshold) is str:
if iterate_threshold.lower() == 'true':
iterate_threshold = True
else:
iterate_threshold = False
if type(adaptive_rmsbox) is str:
if adaptive_rmsbox.lower() == 'true':
adaptive_rmsbox = True
else:
adaptive_rmsbox = False
if type(skip_source_detection) is str:
if skip_source_detection.lower() == 'true':
skip_source_detection = True
else:
skip_source_detection = False
if type(use_adaptive_threshold) is str:
if use_adaptive_threshold.lower() == 'true':
use_adaptive_threshold = True
else:
use_adaptive_threshold = False
if reference_ra_deg is not None and reference_dec_deg is not None:
reference_ra_deg = float(reference_ra_deg)
reference_dec_deg = float(reference_dec_deg)
if not os.path.exists(image_name):
print('Input image not found. Making empty image...')
if not skip_source_detection:
print('ERROR: Source detection cannot be done on an empty image')
sys.exit(1)
if reference_ra_deg is not None and reference_dec_deg is not None:
image_name = mask_name + '.tmp'
make_template_image(image_name,
reference_ra_deg,
reference_dec_deg,
cellsize_deg=float(cellsize_deg))
else:
print(
'ERROR: if image not found, a refernce position must be given')
sys.exit(1)
trim_by = float(trim_by)
atrous_jmax = int(atrous_jmax)
threshpix = float(threshpix)
threshisl = float(threshisl)
nsig = float(nsig)
adaptive_thresh = float(adaptive_thresh)
threshold = 0.0
if not skip_source_detection:
if vertices_file is not None:
# Modify the input image to blank the regions outside of the polygon
temp_img = pim.image(image_name)
image_name += '.blanked'
temp_img.saveas(image_name, overwrite=True)
input_img = pim.image(image_name)
data = input_img.getdata()
vertices = read_vertices(vertices_file)
RAverts = vertices[0]
Decverts = vertices[1]
xvert = []
yvert = []
for RAvert, Decvert in zip(RAverts, Decverts):
pixels = input_img.topixel(
[1, 1, Decvert * np.pi / 180.0, RAvert * np.pi / 180.0])
xvert.append(pixels[2]) # x -> Dec
yvert.append(pixels[3]) # y -> RA
poly = Polygon(xvert, yvert)
# Find masked regions
masked_ind = np.where(data[0, 0])
# Find distance to nearest poly edge and set to NaN those that
# are outside the facet (dist < 0)
dist = poly.is_inside(masked_ind[0], masked_ind[1])
outside_ind = np.where(dist < 0.0)
if len(outside_ind[0]) > 0:
data[0, 0, masked_ind[0][outside_ind],
masked_ind[1][outside_ind]] = np.nan
# Save changes
input_img.putdata(data)
if use_adaptive_threshold:
# Get an estimate of the rms
img = bdsf.process_image(image_name,
mean_map='zero',
rms_box=rmsbox,
thresh_pix=threshpix,
thresh_isl=threshisl,
atrous_do=atrous_do,
ini_method='curvature',
thresh='hard',
adaptive_rms_box=adaptive_rmsbox,
adaptive_thresh=adaptive_thresh,
rms_box_bright=rmsbox_bright,
rms_map=True,
quiet=True,
atrous_jmax=atrous_jmax,
stop_at='isl')
# Find min and max pixels
max_neg_val = abs(np.min(img.ch0_arr))
max_neg_pos = np.where(img.ch0_arr == np.min(img.ch0_arr))
max_pos_val = abs(np.max(img.ch0_arr))
max_pos_pos = np.where(img.ch0_arr == np.max(img.ch0_arr))
# Estimate new thresh_isl from min pixel value's sigma, but don't let
# it get higher than 1/2 of the peak's sigma
threshisl_neg = 2.0 * max_neg_val / img.rms_arr[max_neg_pos][0]
max_sigma = max_pos_val / img.rms_arr[max_pos_pos][0]
if threshisl_neg > max_sigma / 2.0:
threshisl_neg = max_sigma / 2.0
# Use the new threshold only if it is larger than the user-specified one
if threshisl_neg > threshisl:
threshisl = threshisl_neg
if iterate_threshold:
# Start with given threshold and lower it until we get at least one island
nisl = 0
while nisl == 0:
img = bdsf.process_image(image_name,
mean_map='zero',
rms_box=rmsbox,
thresh_pix=threshpix,
thresh_isl=threshisl,
atrous_do=atrous_do,
ini_method='curvature',
thresh='hard',
adaptive_rms_box=adaptive_rmsbox,
adaptive_thresh=adaptive_thresh,
rms_box_bright=rmsbox_bright,
rms_map=True,
quiet=True,
atrous_jmax=atrous_jmax)
nisl = img.nisl
threshpix /= 1.2
threshisl /= 1.2
if threshpix < 5.0:
break
else:
img = bdsf.process_image(image_name,
mean_map='zero',
rms_box=rmsbox,
thresh_pix=threshpix,
thresh_isl=threshisl,
atrous_do=atrous_do,
ini_method='curvature',
thresh='hard',
adaptive_rms_box=adaptive_rmsbox,
adaptive_thresh=adaptive_thresh,
rms_box_bright=rmsbox_bright,
rms_map=True,
quiet=True,
atrous_jmax=atrous_jmax)
if img.nisl == 0:
if region_file is None or region_file == '[]':
print('No islands found. Clean mask cannot be made.')
sys.exit(1)
else:
# Continue on and use user-supplied region file
skip_source_detection = True
threshold = nsig * img.clipped_rms
# Check if there are large islands preset (indicating that multi-scale
# clean is needed)
has_large_isl = False
for isl in img.islands:
if isl.size_active > 100:
# Assuming normal sampling, a size of 100 pixels would imply
# a source of ~ 10 beams
has_large_isl = True
if (region_file is not None and region_file != '[]'
and skip_source_detection):
# Copy region file and return if source detection was not done
os.system('cp {0} {1}'.format(region_file.strip('[]"'), mask_name))
if threshold_format == 'float':
return {'threshold_5sig': threshold}
elif threshold_format == 'str_with_units':
# This is done to get around the need for quotes around strings in casapy scripts
# 'casastr/' is removed by the generic pipeline
return {'threshold_5sig': 'casastr/{0}Jy'.format(threshold)}
elif not skip_source_detection:
img.export_image(img_type='island_mask',
mask_dilation=0,
outfile=mask_name,
img_format=img_format,
clobber=True)
if (vertices_file is not None or trim_by > 0 or pad_to_size is not None
or (region_file is not None and region_file != '[]')
or skip_source_detection):
# Alter the mask in various ways
if skip_source_detection:
# Read the image
mask_im = pim.image(image_name)
else:
# Read the PyBDSF mask
mask_im = pim.image(mask_name)
data = mask_im.getdata()
coordsys = mask_im.coordinates()
if reference_ra_deg is not None and reference_dec_deg is not None:
values = coordsys.get_referencevalue()
values[2][0] = reference_dec_deg / 180.0 * np.pi
values[2][1] = reference_ra_deg / 180.0 * np.pi
coordsys.set_referencevalue(values)
imshape = mask_im.shape()
del (mask_im)
if pad_to_size is not None:
imsize = pad_to_size
coordsys['direction'].set_referencepixel([imsize / 2, imsize / 2])
pixmin = (imsize - imshape[2]) / 2
if pixmin < 0:
print("The padded size must be larger than the original size.")
sys.exit(1)
pixmax = pixmin + imshape[2]
data_pad = np.zeros((1, 1, imsize, imsize), dtype=np.float32)
data_pad[0, 0, pixmin:pixmax, pixmin:pixmax] = data[0, 0]
new_mask = pim.image('',
shape=(1, 1, imsize, imsize),
coordsys=coordsys)
new_mask.putdata(data_pad)
else:
new_mask = pim.image('', shape=imshape, coordsys=coordsys)
new_mask.putdata(data)
data = new_mask.getdata()
if skip_source_detection:
# Mask all pixels
data[:] = 1
if vertices_file is not None:
# Modify the clean mask to exclude regions outside of the polygon
vertices = read_vertices(vertices_file)
RAverts = vertices[0]
Decverts = vertices[1]
xvert = []
yvert = []
for RAvert, Decvert in zip(RAverts, Decverts):
try:
pixels = new_mask.topixel([
0, 1, Decvert * np.pi / 180.0, RAvert * np.pi / 180.0
])
except:
pixels = new_mask.topixel([
1, 1, Decvert * np.pi / 180.0, RAvert * np.pi / 180.0
])
xvert.append(pixels[2]) # x -> Dec
yvert.append(pixels[3]) # y -> RA
poly = Polygon(xvert, yvert)
# Find masked regions
masked_ind = np.where(data[0, 0])
# Find distance to nearest poly edge and unmask those that
# are outside the facet (dist < 0)
dist = poly.is_inside(masked_ind[0], masked_ind[1])
outside_ind = np.where(dist < 0.0)
if len(outside_ind[0]) > 0:
data[0, 0, masked_ind[0][outside_ind],
masked_ind[1][outside_ind]] = 0
if trim_by > 0.0:
sh = np.shape(data)
margin = int(sh[2] * trim_by / 2.0)
data[0, 0, 0:sh[2], 0:margin] = 0
data[0, 0, 0:margin, 0:sh[3]] = 0
data[0, 0, 0:sh[2], sh[3] - margin:sh[3]] = 0
data[0, 0, sh[2] - margin:sh[2], 0:sh[3]] = 0
if region_file is not None and region_file != '[]':
# Merge the CASA regions with the mask
casa_polys = read_casa_polys(region_file.strip('[]"'), new_mask)
for poly in casa_polys:
# Find unmasked regions
unmasked_ind = np.where(data[0, 0] == 0)
# Find distance to nearest poly edge and mask those that
# are inside the casa region (dist > 0)
dist = poly.is_inside(unmasked_ind[0], unmasked_ind[1])
inside_ind = np.where(dist > 0.0)
if len(inside_ind[0]) > 0:
data[0, 0, unmasked_ind[0][inside_ind],
unmasked_ind[1][inside_ind]] = 1
# Save changes
new_mask.putdata(data)
if img_format == 'fits':
new_mask.tofits(mask_name, overwrite=True)
elif img_format == 'casa':
new_mask.saveas(mask_name, overwrite=True)
else:
print(
'Output image format "{}" not understood.'.format(img_format))
sys.exit(1)
if not skip_source_detection:
if threshold_format == 'float':
return {
'threshold_5sig': nsig * img.clipped_rms,
'multiscale': has_large_isl
}
elif threshold_format == 'str_with_units':
# This is done to get around the need for quotes around strings in casapy scripts
# 'casastr/' is removed by the generic pipeline
return {
'threshold_5sig':
'casastr/{0}Jy'.format(nsig * img.clipped_rms),
'multiscale': has_large_isl
}
else:
return {'threshold_5sig': '0.0'}
#!/usr/bin/python
import bdsf
import sys
f = sys.argv[1]
img = bdsf.process_image(f, advanced_opts=True, detection_image=f.replace('-pb',''), interactive=True, thresh_pix=5., thresh_isl=3., \
adaptive_rms_box=True, rms_box_bright=(100,30), adaptive_thresh=10.)
img.write_catalog(format='fits',catalog_type='srl', clobber=True)
img.write_catalog(format='ds9',catalog_type='srl', clobber=True)
img.write_catalog(format='ds9',catalog_type='gaul', clobber=True)
img.export_image(outfile=f+'_gaus_resid.fits', img_type='gaus_resid',clobber=True)
img.export_image(outfile=f+'_gaus_model.fits', img_type='gaus_model',clobber=True)
img.export_image(outfile=f+'_rms.fits', img_type='rms',clobber=True)
img.export_image(outfile=f+'_mean.fits', img_type='mean',clobber=True)
def main(input_image, input_skymodel_pb, input_bright_skymodel_pb, output_root,
vertices_file, threshisl=5.0, threshpix=7.5, rmsbox=(150, 50),
rmsbox_bright=(35, 7), adaptive_rmsbox=True,
use_adaptive_threshold=False, adaptive_thresh=75.0, beamMS=None,
peel_bright=False):
"""
Filter the input sky model so that they lie in islands in the image
Parameters
----------
input_image : str
Filename of input image to use to detect sources for filtering. Ideally, this
should be a flat-noise image (i.e., without primary-beam correction)
input_skymodel_pb : str
Filename of input makesourcedb sky model, with primary-beam correction
input_bright_skymodel_pb : str
Filename of input makesourcedb sky model of bright sources only, with primary-
beam correction
output_root : str
Root of filename of output makesourcedb sky models. Output filenames will be
output_root+'.apparent_sky.txt' and output_root+'.true_sky.txt'
vertices_file : str
Filename of file with vertices
threshisl : float, optional
Value of thresh_isl PyBDSF parameter
threshpix : float, optional
Value of thresh_pix PyBDSF parameter
rmsbox : tuple of floats, optional
Value of rms_box PyBDSF parameter
rmsbox_bright : tuple of floats, optional
Value of rms_box_bright PyBDSF parameter
adaptive_rmsbox : tuple of floats, optional
Value of adaptive_rms_box PyBDSF parameter
use_adaptive_threshold : bool, optional
If True, use an adaptive threshold estimated from the negative values in
the image
adaptive_thresh : float, optional
If adaptive_rmsbox is True, this value sets the threshold above
which a source will use the small rms box
peel_bright : bool, optional
If True, bright sources were peeled, so add then back before filtering
"""
if rmsbox is not None and isinstance(rmsbox, str):
rmsbox = eval(rmsbox)
if isinstance(rmsbox_bright, str):
rmsbox_bright = eval(rmsbox_bright)
adaptive_rmsbox = misc.string2bool(adaptive_rmsbox)
use_adaptive_threshold = misc.string2bool(use_adaptive_threshold)
if isinstance(beamMS, str):
beamMS = misc.string2list(beamMS)
peel_bright = misc.string2bool(peel_bright)
# Try to set the TMPDIR evn var to a short path, to ensure we do not hit the length
# limits for socket paths (used by the mulitprocessing module). We try a number of
# standard paths (the same ones used in the tempfile Python library)
old_tmpdir = os.environ["TMPDIR"]
for tmpdir in ['/tmp', '/var/tmp', '/usr/tmp']:
if os.path.exists(tmpdir):
os.environ["TMPDIR"] = tmpdir
break
# Run PyBDSF to make a mask for grouping
if use_adaptive_threshold:
# Get an estimate of the rms by running PyBDSF to make an rms map
img = bdsf.process_image(input_image, mean_map='zero', rms_box=rmsbox,
thresh_pix=threshpix, thresh_isl=threshisl,
thresh='hard', adaptive_rms_box=adaptive_rmsbox,
adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright,
rms_map=True, quiet=True, stop_at='isl')
# Find min and max pixels
max_neg_val = abs(np.min(img.ch0_arr))
max_neg_pos = np.where(img.ch0_arr == np.min(img.ch0_arr))
max_pos_val = abs(np.max(img.ch0_arr))
max_pos_pos = np.where(img.ch0_arr == np.max(img.ch0_arr))
# Estimate new thresh_isl from min pixel value's sigma, but don't let
# it get higher than 1/2 of the peak's sigma
threshisl_neg = 2.0 * max_neg_val / img.rms_arr[max_neg_pos][0]
max_sigma = max_pos_val / img.rms_arr[max_pos_pos][0]
if threshisl_neg > max_sigma / 2.0:
threshisl_neg = max_sigma / 2.0
# Use the new threshold only if it is larger than the user-specified one
if threshisl_neg > threshisl:
threshisl = threshisl_neg
img = bdsf.process_image(input_image, mean_map='zero', rms_box=rmsbox,
thresh_pix=threshpix, thresh_isl=threshisl,
thresh='hard', adaptive_rms_box=adaptive_rmsbox,
adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright,
atrous_do=True, atrous_jmax=3, rms_map=True, quiet=True)
emptysky = False
if img.nisl > 0:
maskfile = input_image + '.mask'
img.export_image(outfile=maskfile, clobber=True, img_type='island_mask')
# Construct polygon needed to trim the mask to the sector
header = pyfits.getheader(maskfile, 0)
w = wcs.WCS(header)
RAind = w.axis_type_names.index('RA')
Decind = w.axis_type_names.index('DEC')
vertices = misc.read_vertices(vertices_file)
RAverts = vertices[0]
Decverts = vertices[1]
verts = []
for RAvert, Decvert in zip(RAverts, Decverts):
ra_dec = np.array([[0.0, 0.0, 0.0, 0.0]])
ra_dec[0][RAind] = RAvert
ra_dec[0][Decind] = Decvert
verts.append((w.wcs_world2pix(ra_dec, 0)[0][RAind], w.wcs_world2pix(ra_dec, 0)[0][Decind]))
hdu = pyfits.open(maskfile, memmap=False)
data = hdu[0].data
# Rasterize the poly
data_rasertize = data[0, 0, :, :]
data_rasertize = misc.rasterize(verts, data_rasertize)
data[0, 0, :, :] = data_rasertize
hdu[0].data = data
hdu.writeto(maskfile, overwrite=True)
# Now filter the sky model using the mask made above
if len(beamMS) > 1:
# Select the best MS for the beam attenuation
ms_times = []
for ms in beamMS:
tab = pt.table(ms, ack=False)
ms_times.append(np.mean(tab.getcol('TIME')))
tab.close()
ms_times_sorted = sorted(ms_times)
mid_time = ms_times_sorted[int(len(ms_times)/2)]
beam_ind = ms_times.index(mid_time)
else:
beam_ind = 0
try:
s = lsmtool.load(input_skymodel_pb, beamMS=beamMS[beam_ind])
except astropy.io.ascii.InconsistentTableError:
emptysky = True
if peel_bright:
try:
# If bright sources were peeled before imaging, add them back
s_bright = lsmtool.load(input_bright_skymodel_pb, beamMS=beamMS[beam_ind])
# Rename the bright sources, removing the '_sector_*' added previously
# (otherwise the '_sector_*' text will be added every iteration,
# eventually making for very long source names)
new_names = [name.split('_sector')[0] for name in s_bright.getColValues('Name')]
s_bright.setColValues('Name', new_names)
if not emptysky:
s.concatenate(s_bright)
else:
s = s_bright
emptysky = False
except astropy.io.ascii.InconsistentTableError:
pass
if not emptysky:
s.select('{} == True'.format(maskfile)) # keep only those in PyBDSF masked regions
if len(s) == 0:
emptysky = True
else:
# Write out apparent and true-sky models
del(img) # helps reduce memory usage
s.group(maskfile) # group the sky model by mask islands
s.write(output_root+'.true_sky.txt', clobber=True)
s.write(output_root+'.apparent_sky.txt', clobber=True, applyBeam=True)
else:
emptysky = True
if emptysky:
# No sources cleaned/found in image, so just make a dummy sky model with single,
# very faint source at center
dummylines = ["Format = Name, Type, Patch, Ra, Dec, I, SpectralIndex, LogarithmicSI, "
"ReferenceFrequency='100000000.0', MajorAxis, MinorAxis, Orientation\n"]
ra, dec = img.pix2sky((img.shape[-2]/2.0, img.shape[-1]/2.0))
if ra < 0.0:
ra += 360.0
ra = misc.ra2hhmmss(ra)
sra = str(ra[0]).zfill(2)+':'+str(ra[1]).zfill(2)+':'+str("%.6f" % (ra[2])).zfill(6)
dec = misc.dec2ddmmss(dec)
decsign = ('-' if dec[3] < 0 else '+')
sdec = decsign+str(dec[0]).zfill(2)+'.'+str(dec[1]).zfill(2)+'.'+str("%.6f" % (dec[2])).zfill(6)
dummylines.append(',,p1,{0},{1}\n'.format(sra, sdec))
dummylines.append('s0c0,POINT,p1,{0},{1},0.00000001,'
'[0.0,0.0],false,100000000.0,,,\n'.format(sra, sdec))
with open(output_root+'.apparent_sky.txt', 'w') as f:
f.writelines(dummylines)
with open(output_root+'.true_sky.txt', 'w') as f:
f.writelines(dummylines)
# Set the TMPDIR env var back to its original value
os.environ["TMPDIR"] = old_tmpdir
def run_bdsf(image, output_dir, argfile, output_format):
'''
Run PyBDSF on an image
Keyword arguments:
image -- Name of image
argfile -- Input json file containing arguments
for bdsf functions
'''
imname = os.path.join(output_dir, os.path.basename(image).split('.')[0])
path = Path(__file__).parent / argfile
with open(path) as f:
args_dict = json.load(f)
# Fix json stupidness
args_dict['process_image']['rms_box'] = ast.literal_eval(
args_dict['process_image']['rms_box'])
args_dict['process_image']['rms_box_bright'] = ast.literal_eval(
args_dict['process_image']['rms_box_bright'])
img = bdsf.process_image(image, **args_dict['process_image'])
for img_type in args_dict['export_image']:
if args_dict['export_image'][img_type]:
img.export_image(outfile=imname + '_' + img_type + '.fits',
clobber=True,
img_type=img_type)
outcat = None
for of in output_format:
fmt = of.lower().split(':')
if len(fmt) == 1:
fmt = fmt[0]
cat_type = 'srl'
if len(fmt) == 2:
fmt, cat_type = fmt
if fmt == 'ds9':
outcatalog = imname + '_' + cat_type + '_bdsfcat.ds9.reg'
img.write_catalog(outfile=outcatalog,
format=fmt,
catalog_type=cat_type,
clobber=True)
elif fmt == 'kvis':
outcatalog = imname + '_bdsfcat.kvis.ann'
img.write_catalog(outfile=outcatalog,
format=fmt,
catalog_type='gaul',
clobber=True)
elif fmt == 'star':
outcatalog = imname + '_bdsfcat.star'
img.write_catalog(outfile=outcatalog,
format=fmt,
catalog_type='gaul',
clobber=True)
else:
outcatalog = imname + '_' + cat_type + '_bdsfcat.' + fmt
img.write_catalog(outfile=outcatalog,
format=fmt,
catalog_type=cat_type,
clobber=True)
if fmt == 'fits' and cat_type == 'srl':
outcat = outcatalog
return outcat, img
# configure trimbox
if args.trimbox is not None:
with fits.open(fname) as hdu:
naxis1 = hdu[0].header["NAXIS1"]
naxis2 = hdu[0].header["NAXIS1"]
trim_box = (int(naxis1 / 2) - args.trimbox, int(naxis1 / 2) +
args.trimbox, int(naxis2 / 2) - args.trimbox,
int(naxis2 / 2) + args.trimbox)
else:
trim_box = None
# run bdsf
try:
img = bdsf.process_image(fname,
quiet=args.silence,
trim_box=trim_box,
blank_limit=args.blank_limit)
except RuntimeError:
continue
# save catalog to ASCII file
outfiles.append(out_fname)
echo("::: saving {} :::".format(out_fname))
bdsf.output.write_ascii_list(img,
filename=out_fname,
clobber=args.overwrite)
# stop if no source provided
if args.source is None:
echo(
"can't proceed with source extraction because args.source is not provided"
def main (vis,strategy='P30,P30,P30,A500,A450,A400',startmod='',ith=5.0,\
bandwidth='8MHz',goodness=2.,minuvw=50.0,robust=-1.0):
## format arguments
strategy = str(strategy)
startmod = str(startmod)
ith = float(ith)
bandwidth = str(bandwidth)
minuvw = float(minuvw) * 1000.0 # convert km -> m
robust = float(robust)
## process arguments
vis = vis.rstrip('/')
vis = vis.split('/')[-1]
strategy = strategy.split(',')
bw_val = ''
bw_unit = ''
for c in bandwidth:
try:
float(c)
bw_val = bw_val + c
except ValueError:
bw_unit = bw_unit + c
if bw_unit == 'MHz':
bw_val = float(bw_val) * 1e6
## get bandwidth of vis
spec_info = casatb.table(vis + '::SPECTRAL_WINDOW')
total_bw = spec_info.getcol('TOTAL_BANDWIDTH')[0]
num_chan = spec_info.getcol('NUM_CHAN')[0]
spec_info.close()
if total_bw < bw_val:
wsclean_chans = 0
mfs = ''
nchan = 0
else:
wsclean_chans = int(np.ceil(total_bw / bw_val))
mfs = '-MFS'
nchan = num_chan / wsclean_chans
## make a working directory and go there
tmp_dir = 'loop3_' + vis.rstrip('.ms').rstrip('.MS')
os.system('mkdir %s' % tmp_dir)
os.chdir(tmp_dir)
os.system('mv ../%s .' % vis)
import bdsf
prevstat = 0.0
cohlength = 2.0E6
strategy_type = []
for i in strategy:
strategy_type.append(i[0])
ploop, nloop, snver = strategy_type.count('P'), len(strategy), 0
#
# PHASE CALIBRATION - run through ploop iterations, exiting if we have convergence
#
for iloop in range(ploop):
fitsmask = vis + '_%02d-mask.fits' % (iloop - 1) if iloop else ''
if startmod == '' or iloop:
pstr = '******* PHASE LOOP %d running wsclean ************' % iloop
loop3log(vis, pstr + '\n')
imagr(vis,minuvwm=minuvw,robust=robust,cellsize='0.05asec',domultiscale=True,\
outname=vis+'_%02d'%iloop,channelsout=wsclean_chans,\
fitsmask=fitsmask,dolocalrms=True,maxuvwm=cohlength)
else:
# Need something here to produce an image from startmod
pass
# check if there's a source
thisstat = measure_statistic2(vis + '_%02d%s-image.fits' %
(iloop, mfs))
if thisstat < goodness:
pstr = 'SNR is %f, breaking out of loop.' % thisstat
loop3log(vis, pstr + '\n')
montage_plot('*MFS-image.fits',
imscale=0.65,
nup='4x2',
plot_resid=False)
return (0)
pstr = '******* PHASE LOOP %d making mask %s_%02d%s-image.fits ********' % (
iloop, vis, iloop, mfs)
loop3log(vis, pstr + '\n')
stdout = sys.stdout
sys.stdout = open('bdsf_chunterings', 'a')
img = bdsf.process_image('%s_%02d%s-image.fits' % (vis, iloop, mfs),
atrous_do=True,
thresh_isl=ith)
sys.stdout.close()
sys.stdout = stdout
img.export_image(img_type='island_mask',
outfile='%s_%02d-mask.fits' % (vis, iloop))
# exit loop if clean finishing
pstr = '******* PHASE LOOP %d goodness stat %f ************' % (
iloop, thisstat)
loop3log(vis, pstr + '\n')
if thisstat - prevstat < 0.01:
pstr = '****** EXITING PHASE CAL with diff %f *********' % (
thisstat - prevstat)
loop3log(vis, pstr + '\n')
break
else:
prevstat = thisstat
imagr(vis,minuvwm=minuvw,robust=robust,dopredict=True,fitsmask=fitsmask,\
autothreshold=3,dolocalrms=True,\
outname=vis+'_%02d%s'%(iloop,mfs))
pstr = '******* PHASE LOOP %d making new cal file %s ************' % (
iloop, vis + '_%02d' % iloop)
loop3log(vis, pstr + '\n')
caltype, sol0 = strategy[iloop][0], float(strategy[iloop][1:])
coh, cohlength = selfcal(vis,minuvw,robust,model='MODEL',incol='DATA',\
outcol='CORRECTED_DATA',outcal_root=vis+'_%02d'%iloop,\
caltype=caltype,init_sol=sol0,nchan=nchan)
snver = iloop
pstr='******** END PHASE LOOP %d - coherence on %.1f km **********' % \
(iloop,cohlength/1000.)
loop3log(vis, pstr + '\n')
# Exit at this point if we are not doing amplitude cal
if ploop == nloop:
exit()
#
# If we are doing amplitude calibration, we now need to apply the
# calibration and write a new MS with a DATA column
visA = vis + '_A'
# delete all existing files beginning with vis+'_A'
os.system('rm -fr %s*' % visA)
pstr = '****** APPLYING CALIBRATION TABLE %d\n' % snver
loop3log(vis, pstr + '\n')
applycal_split(vis, visA, 'sol000', '%s_%02d_c0.h5' % (vis, snver))
init_fitsmask = vis + '_%02d-mask.fits' % iloop
init_img = vis + '_%02d%s-image.fits' % (iloop, mfs)
pred_img = vis + '_%02d%s' % (iloop, mfs)
for iloop in range(ploop, nloop):
fitsmask = init_fitsmask if iloop == ploop else visA + '_%02d-mask.fits' % (
iloop - 1)
pstr = '******* AMPLITUDE LOOP %d running wsclean ************' % iloop
loop3log(vis, pstr + '\n')
imagr(visA,minuvwm=minuvw,robust=robust,cellsize='0.05asec',domultiscale=True,\
outname=visA+'_%02d'%iloop,channelsout=wsclean_chans,\
fitsmask=fitsmask,dolocalrms=True,maxuvwm=cohlength)
## check if there's a source
thisstat = measure_statistic2(visA + '_%02d%s-image.fits' %
(iloop, mfs))
if thisstat < goodness:
pstr = 'SNR is %f, breaking out of loop.' % thisstat
loop3log(vis, pstr + '\n')
montage_plot('*MFS-image.fits',
imscale=0.65,
nup='4x2',
plot_resid=True)
return (0)
image_bdsf = '%s_%02d%s-image.fits' % (visA, iloop, mfs)
pstr = '******* AMPLITUDE LOOP %d making mask %s_%02d%s-image.fits ************' % (
iloop, visA, iloop, mfs)
loop3log(vis, pstr + '\n')
img = bdsf.process_image(image_bdsf, atrous_do=True, thresh_isl=ith)
img.export_image(img_type='island_mask',
outfile='%s_%02d-mask.fits' % (visA, iloop))
pstr = '******* AMPLITUDE LOOP %d goodness stat %f ************' % (
iloop, thisstat)
loop3log(vis, pstr + '\n')
if iloop != ploop and thisstat - prevstat < 0.01:
pstr = '****** EXITING AMPLITUDE CAL with diff %f *********' % (
thisstat - prevstat)
loop3log(vis, pstr + '\n')
break
else:
prevstat = thisstat
imagr(visA,minuvwm=minuvw,dopredict=True,fitsmask=fitsmask,\
autothreshold=3,dolocalrms=True,robust=robust,\
outname=visA+'_%02d%s'%(iloop,mfs))
pstr = '******* AMPLITUDE LOOP %d making new cal file %s ************' % (
iloop, visA + '_%02d' % iloop)
loop3log(vis, pstr + '\n')
caltype, sol0 = strategy[iloop][0], float(strategy[iloop][1:])
coh,cohlength = selfcal(visA,minuvw,robust,model='MODEL',incol='DATA',\
outcol='CORRECTED_DATA',outcal_root=visA+'_%02d'%iloop,\
caltype=caltype,init_sol=sol0,nchan=nchan)
pstr='******** END AMPLITUDE LOOP %d - coherence on %.1f km **********' % \
(iloop,cohlength/1000.)
loop3log(vis, pstr + '\n')
fitsmask = init_fitsmask if iloop == ploop else visA + '_%02d-mask.fits' % (
iloop - 1)
imagr(visA,minuvwm=minuvw,cellsize='0.05asec',domultiscale=True,\
outname=visA+'_final',channelsout=wsclean_chans,robust=robust,\
fitsmask=fitsmask,dolocalrms=True)
## make a model from the final image
final_im = glob.glob('*final*image.fits')
if len(final_im) > 1:
tmp = [a for a in final_im if 'MFS' in a]
final_im = tmp
img = bdsf.process_image(final_im[0], atrous_do=True, thresh_isl=ith)
skyfile = final_im[0].replace('fits', 'skymodel')
img.write_catalog(outfile=skyfile,
bbs_patches='single',
catalog_type='gaul',
format='bbs')
## convert it to a sourcedb
ss = "makesourcedb in=%s out=%s format='<'" % (
skyfile, skyfile.replace('skymodel', 'sky'))
os.system(ss)
## plot things like solutions
make_plots(vis)
make_plots(visA)
## If we got to this point, self-cal has successfully completed
# montage_plot( '*MFS-image.fits', imscale=0.65, nup='4x2', plot_resid=True)
pngfile, h5files = cleanup(vis)
for h5file in h5files:
os.system('mv %s ../' % h5file)
# os.system('mv *.pdf ../')
# os.system('mv *.png ../')
os.system('mv *skymodel ../')
os.system('mv *sky ../')
os.system('mv %s ../' % vis)
print 'Output calibration tables', h5files
#return pngfile,h5files
return 0
def aplpy_plots(infits,
docut=2.0,
outpng='',
nolabel=False,
crms=3.0,
noshift=False,
margin=1.7):
## open the file to get the data and the header information
print 'Plotting ', infits
hdul = pyfits.open(infits)
a = hdul[0].data.squeeze()
h = hdul[0].header
nx, ny = h['NAXIS1'], h['NAXIS2']
field_radius = h['CDELT2'] * ny / 2.0
trms, tmax = np.array([]), np.array([])
for i in range(10):
x1, x2 = int(0.1 * i * nx), int(0.1 * (i + 1) * nx - 1)
for j in range(10):
y1, y2 = int(0.1 * j * ny), int(0.1 * (j + 1) * ny - 1)
trms = np.append(trms, np.std(a[y1:y2, x1:x2]))
tmax = np.append(tmax, np.std(a[y1:y2, x1:x2]))
rms = np.nanmedian(trms)
vmin, vmax = np.nanmin(a), np.nanmax(a)
## calculate the required sigma
reqsig = stats.norm.ppf(1 - 0.5 / float(nx * ny)) + (2.0 if docut < -1.0
else docut)
## run bdsf to get a gaussian list
img = bdsf.process_image(infits, thresh_isl=reqsig)
img.write_catalog(outfile='temp.gaul',
catalog_type='gaul',
format='ascii',
clobber=True)
## read in the file
s = ascii.read('temp.gaul', format='commented_header', header_start=4)
print s
## make a plot
gc = aplpy.FITSFigure(infits)
pixra, pixdec = np.mean(s['Xposn']), np.mean(s['Yposn'])
dec = h['CRVAL2'] + h['CDELT2'] * (pixdec - h['CRPIX2'])
cosdec = np.cos(np.deg2rad(dec))
ra = h['CRVAL1'] + h['CDELT1'] * (pixra - h['CRPIX1']) / cosdec
deccen = h['CRVAL2'] + h['CDELT2'] * (0.5 * ny - h['CRPIX2'])
cosdeccen = np.cos(np.deg2rad(deccen))
racen = h['CRVAL1'] + h['CDELT1'] * (0.5 * nx - h['CRPIX1']) / cosdeccen
pix_range = max(s['Xposn'].max() - s['Xposn'].min(),
s['Yposn'].max() - s['Yposn'].min())
try:
deg_range = max(margin * h['CDELT2'] * pix_range,
0.1 * ny * h['CDELT2'])
except:
deg_range = 1. / 60.
print 'Error on deg_range, pix_range %f' % pix_range
if docut < -1.0:
deg_range = -2.0 * docut / 3600.
print 'Range is %.1f arcsec, %.1f pix\n' % (1800.0 * deg_range, pix_range)
if noshift:
gc.recenter(racen, deccen, 0.5 * deg_range)
else:
gc.recenter(ra, dec, 0.5 * deg_range)
gc.set_tick_color('black')
gc.show_colorscale(cmap=matplotlib.cm.gray_r, vmin=vmin, vmax=vmax)
levels, tlevels = [vmax], vmax
while tlevels > crms * rms:
tlevels /= np.sqrt(2)
levels.append(tlevels)
gc.show_contour(levels=np.sort(levels))
if not nolabel:
bstr = ''
for i in range(len(h)):
try:
if 'BMAJ' in h[i] and 'AIPS' in h[i]:
bmaj = 3600. * float(h[i].split('BMAJ=')[1].split()[0])
bmin = 3600. * float(h[i].split('BMIN=')[1].split()[0])
bstr = 'beam %.1fx%.1f' % (bmaj, bmin)
except:
pass
if rms > 0.1:
gc.add_label(0.5,0.05,'Peak %.1f, rms %.1f Jy %s'%\
(vmax,rms,bstr),relative=True,size=14)
elif rms > 1.e-4:
gc.add_label(0.5,0.05,'Peak %.1f, rms %.1f mJy %s'%\
(vmax*1000.,rms*1000.,bstr),relative=True,size=14)
else:
gc.add_label(0.5,0.05,'Peak %.1f, rms %.1f uJy %s'%\
(vmax*1.e6,rms*1.e6,bstr),relative=True,size=14)
if outpng == '':
outpng = infits.replace('fits', 'png')
gc.save(outpng)
# Copyright (C) 2019 - Francesco de Gasperin
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import bdsf
import sys
f = sys.argv[1]
img = bdsf.process_image(f, advanced_opts=True, detection_image=f.replace('-pb',''), interactive=True, thresh_pix=5., thresh_isl=3., \
adaptive_rms_box=True, rms_box_bright=(100,30), adaptive_thresh=10.)
img.write_catalog(format='fits',catalog_type='srl', clobber=True)
img.write_catalog(format='ds9',catalog_type='srl', clobber=True)
img.write_catalog(format='ds9',catalog_type='gaul', clobber=True)
img.export_image(outfile=f+'_gaus_resid.fits', img_type='gaus_resid',clobber=True)
img.export_image(outfile=f+'_gaus_model.fits', img_type='gaus_model',clobber=True)
img.export_image(outfile=f+'_rms.fits', img_type='rms',clobber=True)
img.export_image(outfile=f+'_mean.fits', img_type='mean',clobber=True)
def run_bootstrap(o):
colname='DATA_DI_CORRECTED'
if o['mslist'] is None:
die('MS list must be specified')
if o['logging'] is not None and not os.path.isdir(o['logging']):
os.mkdir(o['logging'])
# check the data supplied
if o['frequencies'] is None or o['catalogues'] is None:
die('Frequencies and catalogues options must be specified')
if "DDF_PIPELINE_CATALOGS" not in os.environ.keys():
warn("You need to define the environment variable DDF_PIPELINE_CATALOGS where your catalogs are located")
sys.exit(2)
o["tgss"]=o["tgss"].replace("$$",os.environ["DDF_PIPELINE_CATALOGS"])
o["catalogues"]=[l.replace("$$",os.environ["DDF_PIPELINE_CATALOGS"]) for l in o["catalogues"]]
lCat=o["catalogues"]+[o["tgss"]]
for fCat in lCat:
if not os.path.isfile(fCat):
warn("Catalog %s does not exist"%fCat)
sys.exit(2)
cl=len(o['catalogues'])
if o['names'] is None:
o['names']=[os.path.basename(x).replace('.fits','') for x in o['catalogues']]
if o['radii'] is None:
o['radii']=[10]*cl
if o['groups'] is None:
o['groups']=range(cl)
if (len(o['frequencies'])!=cl or len(o['radii'])!=cl or
len(o['names'])!=cl or len(o['groups'])!=cl):
die('Names, groups, radii and frequencies entries must be the same length as the catalogue list')
low_uvrange=[o['image_uvmin'],2.5*206.0/o['low_psf_arcsec']]
if o['low_imsize'] is not None:
low_imsize=o['low_imsize'] # allow over-ride
else:
low_imsize=o['imsize']*o['cellsize']/o['low_cell']
low_robust=o['low_robust']
# Clear the shared memory
run('CleanSHM.py',dryrun=o['dryrun'])
# We use the individual ms in mslist.
m=MSList(o['mslist'])
Uobsid = set(m.obsids)
for obsid in Uobsid:
warn('Running bootstrap for obsid %s' % obsid)
freqs=[]
omslist=[]
for ms,ob,f in zip(m.mss,m.obsids,m.freqs):
if ob==obsid:
omslist.append(ms)
freqs.append(f)
if len(freqs)<4:
die('Not enough frequencies to bootstrap. Check your mslist or MS naming scheme')
# sort to work in frequency order
freqs,omslist = (list(x) for x in zip(*sorted(zip(freqs, omslist), key=lambda pair: pair[0])))
for f,ms in zip(freqs,omslist):
print ms,f
# generate the sorted input mslist
with open('temp_mslist.txt','w') as f:
for line in omslist:
f.write(line+'\n')
# Clean in cube mode
# As for the main pipeline, first make a dirty map
ddf_image('image_bootstrap_'+obsid+'_init','temp_mslist.txt',
cleanmask=None,cleanmode='SSD',ddsols='DDS0',
applysols='P',majorcycles=0,robust=low_robust,
uvrange=low_uvrange,beamsize=o['low_psf_arcsec'],
imsize=low_imsize,cellsize=o['low_cell'],
options=o,colname=colname,automask=True,
automask_threshold=15,smooth=True,cubemode=True,
conditional_clearcache=True)
external_mask='bootstrap_external_mask.fits'
make_external_mask(external_mask,'image_bootstrap_'+obsid+'_init.dirty.fits',use_tgss=True,clobber=False,cellsize='low_cell',options=o)
# Deep SSD clean with this external mask and automasking
ddf_image('image_bootstrap_'+obsid,'temp_mslist.txt',
cleanmask=external_mask,reuse_psf=True,reuse_dirty=True,
cleanmode='SSD',ddsols='DDS0',applysols='P',
majorcycles=5,robust=low_robust,uvrange=low_uvrange,
beamsize=o['low_psf_arcsec'],imsize=low_imsize,
cellsize=o['low_cell'],options=o,
colname=colname,automask=True,
automask_threshold=15,smooth=True,cubemode=True,
conditional_clearcache=False)
if os.path.isfile('image_bootstrap_'+obsid+'.cube.int.restored.pybdsm.srl'):
warn('Source list exists, skipping source extraction')
else:
warn('Running PyBDSM, please wait...')
img=bdsm.process_image('image_bootstrap_'+obsid+'.cube.int.restored.fits',thresh_pix=5,rms_map=True,atrous_do=True,atrous_jmax=2,group_by_isl=True,rms_box=(80,20), adaptive_rms_box=True, adaptive_thresh=80, rms_box_bright=(35,7),mean_map='zero',spectralindex_do=True,specind_maxchan=1,debug=True,kappa_clip=3,flagchan_rms=False,flagchan_snr=False,incl_chan=True,spline_rank=1)
# Write out in ASCII to work round bug in pybdsm
img.write_catalog(catalog_type='srl',format='ascii',incl_chan='true')
img.export_image(img_type='rms',img_format='fits')
from make_fitting_product import make_catalogue
import fitting_factors
import find_outliers
# generate the fitting product
if os.path.isfile(obsid+'crossmatch-1.fits'):
warn('Crossmatch table exists, skipping crossmatch')
else:
t = pt.table(omslist[0]+ '/FIELD', readonly=True, ack=False)
direction = t[0]['PHASE_DIR']
ra, dec = direction[0]
if (ra<0):
ra+=2*np.pi
ra*=180.0/np.pi
dec*=180.0/np.pi
cats=zip(o['catalogues'],o['names'],o['groups'],o['radii'])
make_catalogue('image_bootstrap_'+obsid+'.cube.int.restored.pybdsm.srl',ra,dec,2.5,cats,outnameprefix=obsid)
freqlist=open(obsid+'frequencies.txt','w')
for n,f in zip(o['names'],o['frequencies']):
freqlist.write('%f %s_Total_flux %s_E_Total_flux False\n' % (f,n,n))
for i,f in enumerate(freqs):
freqlist.write('%f Total_flux_ch%i E_Total_flux_ch%i True\n' % (f,i+1,i+1))
freqlist.close()
# Now call the fitting code
if os.path.isfile(obsid+'crossmatch-results-1.npy'):
warn('Results 1 exists, skipping first fit')
else:
fitting_factors.run_all(1, name=obsid)
nreject=-1 # avoid error if we fail somewhere
if os.path.isfile(obsid+'crossmatch-2.fits'):
warn('Second crossmatch exists, skipping outlier rejection')
else:
nreject=find_outliers.run_all(1, name=obsid)
if os.path.isfile(obsid+'crossmatch-results-2.npy'):
warn('Results 2 exists, skipping second fit')
else:
if nreject==0:
shutil.copyfile(obsid+'crossmatch-results-1.npy',obsid+'crossmatch-results-2.npy')
if os.path.isfile(obsid+'crossmatch-results-2.npy'):
warn('Results 2 exists, skipping first fit')
else:
fitting_factors.run_all(2, name=obsid)
# Now apply corrections
if o['full_mslist'] is None:
die('Need big mslist to apply corrections')
if not(o['dryrun']):
warn('Applying corrections to MS list')
scale=np.load(obsid+'crossmatch-results-2.npy')[:,0]
# InterpolatedUS gives us linear interpolation between points
# and extrapolation outside it
spl = InterpolatedUnivariateSpline(freqs, scale, k=1)
bigmslist=[s.strip() for s in open(o['full_mslist']).readlines()]
obigmslist = [ms for ms in bigmslist if obsid in ms]
for ms in obigmslist:
t = pt.table(ms)
try:
dummy=t.getcoldesc('SCALED_DATA')
except RuntimeError:
dummy=None
t.close()
if dummy is not None:
warn('Table '+ms+' has already been corrected, skipping')
else:
# in this version we need to scale both the original data and the data in colname
t = pt.table(ms+'/SPECTRAL_WINDOW', readonly=True, ack=False)
frq=t[0]['REF_FREQUENCY']
factor=spl(frq)
print frq,factor
t=pt.table(ms,readonly=False)
desc=t.getcoldesc(o['colname'])
desc['name']='SCALED_DATA'
t.addcols(desc)
d=t.getcol(o['colname'])
d*=factor
t.putcol('SCALED_DATA',d)
try:
dummy=t.getcoldesc(colname)
except RuntimeError:
dummy=None
if dummy is not None:
desc=t.getcoldesc(colname)
newname=colname+'_SCALED'
desc['name']=newname
t.addcols(desc)
d=t.getcol(colname)
d*=factor
t.putcol(newname,d)
t.close()
if os.path.isfile('image_bootstrap.app.mean.fits'):
warn('Mean bootstrap image exists, not creating it')
else:
warn('Creating mean bootstrap image')
hdus=[]
for obsid in Uobsid:
hdus.append(fits.open('image_bootstrap_'+obsid+'.app.restored.fits'))
for i in range(1,len(Uobsid)):
hdus[0][0].data+=hdus[i][0].data
hdus[0][0].data/=len(Uobsid)
hdus[0].writeto('image_bootstrap.app.mean.fits')
def hybridloops(vis,
strategy=['P32'] * 3 + ['A1800', 'A1200', 'A600'],
startmod='',
ith=5.0):
import bdsf
prevstat = 0.0
cohlength = 2.0E6
strategy_type = []
for i in strategy:
strategy_type.append(i[0])
ploop, nloop, snver = strategy_type.count('P'), len(strategy), 0
#
# PHASE CALIBRATION - run through ploop iterations, exiting if we have convergence
#
for iloop in range(ploop):
fitsmask = vis + '_%02d-mask.fits' % (iloop - 1) if iloop else ''
if startmod == '' or iloop:
pstr = '******* PHASE LOOP %d running wsclean ************' % iloop
loop3_service.loop3log(vis, pstr + '\n')
loop3_service.imagr(vis,cellsize='0.1asec',domultiscale=True,\
outname=vis+'_%02d'%iloop,dojoinchannels=True,channelsout=8,robust=0,\
fitsmask=fitsmask,dolocalrms=True,maxuvwm=cohlength)
else:
# Need something here to produce an image from startmod
pass
pstr = '******* PHASE LOOP %d making mask %s_%02d-MFA-image.fits ********' % (
iloop, vis, iloop)
loop3_service.loop3log(vis, pstr + '\n')
stdout = sys.stdout
sys.stdout = open('bdsf_chunterings', 'a')
img = bdsf.process_image('%s_%02d-MFS-image.fits' % (vis, iloop),
atrous_do=True,
thresh_isl=ith)
sys.stdout.close()
sys.stdout = stdout
img.export_image(img_type='island_mask',
outfile='%s_%02d-mask.fits' % (vis, iloop))
thisstat = measure_statistic(vis + '_%02d-MFS-image.fits' % iloop)
####### need a line here to bomb out if no coherence
# exit loop if clean finishing
pstr = '******* PHASE LOOP %d goodness stat %f ************' % (
iloop, thisstat)
loop3_service.loop3log(vis, pstr + '\n')
if thisstat - prevstat < 0.01:
pstr = '****** EXITING PHASE CAL with diff %f *********' % (
thisstat - prevstat)
loop3_service.loop3log(vis, pstr + '\n')
break
else:
prevstat = thisstat
loop3_service.imagr(vis,dopredict=True,fitsmask=fitsmask,autothreshold=2.5,dolocalrms=True,\
robust=0,outname=vis+'_%02d-MFS'%iloop)
pstr = '******* PHASE LOOP %d making new cal file %s ************' % (
iloop, vis + '_%02d' % iloop)
loop3_service.loop3log(vis, pstr + '\n')
caltype, sol0 = strategy[iloop][0], float(strategy[iloop][1:])
coh, cohlength = selfcal(vis,model='MODEL',incol='DATA',outcol='CORRECTED_DATA',\
outcal_root=vis+'_%02d'%iloop,caltype=caltype,init_sol=sol0)
snver = iloop
pstr='******** END PHASE LOOP %d - coherence on %.1f km **********' % \
(iloop,cohlength/1000.)
loop3_service.loop3log(vis, pstr + '\n')
# Exit at this point if we are not doing amplitude cal
if ploop == nloop:
exit()
#
# If we are doing amplitude calibration, we now need to apply the
# calibration and write a new MS with a DATA column
visA = vis + '_A'
# delete all existing files beginning with vis+'_A'
os.system('rm -fr %s*' % visA)
pstr = '****** APPLYING CALIBRATION TABLE %d\n' % snver
loop3_service.loop3log(vis, pstr + '\n')
applycal_split(vis, visA, 'sol000', '%s_%02d_c0.h5' % (vis, snver))
init_fitsmask = vis + '_%02d-mask.fits' % iloop
init_img = vis + '_%02d-MFS-image.fits' % iloop
pred_img = vis + '_%02d-MFS' % iloop
for iloop in range(ploop, nloop):
fitsmask = init_fitsmask if iloop == ploop else visA + '_%02d-mask.fits' % (
iloop - 1)
pstr = '******* AMPLITUDE LOOP %d running wsclean ************' % iloop
loop3_service.loop3log(vis, pstr + '\n')
loop3_service.imagr(visA,cellsize='0.1asec',domultiscale=True,\
outname=visA+'_%02d'%iloop,dojoinchannels=True,channelsout=8,robust=0,\
fitsmask=fitsmask,dolocalrms=True,maxuvwm=cohlength)
image_bdsf = '%s_%02d-MFS-image.fits' % (visA, iloop)
pstr = '******* AMPLITUDE LOOP %d making mask %s_%02d-MFS-image.fits ************' % (
iloop, visA, iloop)
loop3_service.loop3log(vis, pstr + '\n')
img = bdsf.process_image(image_bdsf, atrous_do=True, thresh_isl=5.0)
img.export_image(img_type='island_mask',
outfile='%s_%02d-mask.fits' % (visA, iloop))
thisstat = measure_statistic(visA + '_%02d-MFS-image.fits' % iloop)
pstr = '******* AMPLITUDE LOOP %d goodness stat %f ************' % (
iloop, thisstat)
loop3_service.loop3log(vis, pstr + '\n')
if iloop != ploop and thisstat - prevstat < 0.01:
pstr = '****** EXITING AMPLITUDE CAL with diff %f *********' % (
thisstat - prevstat)
loop3_service.loop3log(vis, pstr + '\n')
break
else:
prevstat = thisstat
loop3_service.imagr(visA,dopredict=True,fitsmask=fitsmask,autothreshold=2.5,dolocalrms=True,\
robust=0,outname=visA+'_%02d-MFS'%iloop)
pstr = '******* AMPLITUDE LOOP %d making new cal file %s ************' % (
iloop, visA + '_%02d' % iloop)
loop3_service.loop3log(vis, pstr + '\n')
caltype, sol0 = strategy[iloop][0], float(strategy[iloop][1:])
coh,cohlength = selfcal(visA,model='MODEL',incol='DATA',outcol='CORRECTED_DATA',\
outcal_root=visA+'_%02d'%iloop,caltype=caltype,init_sol=sol0)
pstr='******** END AMPLITUDE LOOP %d - coherence on %.1f km **********' % \
(iloop,cohlength/1000.)
loop3_service.loop3log(vis, pstr + '\n')
fitsmask = init_fitsmask if iloop == ploop else visA + '_%02d-mask.fits' % (
iloop - 1)
loop3_service.imagr(visA,cellsize='0.1asec',domultiscale=True,\
outname=visA+'_final',dojoinchannels=True,channelsout=8,robust=0,\
fitsmask=fitsmask,dolocalrms=True)
loop3_service.montage_plot(vis)
pngfile, h5files = cleanup(vis)
loop3_service.loop3log(vis, 'Output png file %s' % pngfile)
print 'Output calibration tables', h5files
return pngfile, h5files
def do_offsets(o):
# o is the options file
if o['mode'] != 'normal' and o['mode'] != 'test':
raise NotImplementedError('Offsets called with mode ' + o['mode'])
image_root = 'image_full_ampphase_di_m.NS'
method = o['method']
report('Determining astrometric offsets with method ' + method +
' in mode ' + o['mode'])
report('Merging downloaded catalogues')
if os.path.isfile(method + '.fits'):
warn('Merged file exists, reading from disk instead')
data = Table.read(method + '.fits')
else:
if method == 'pslocal':
data = Table.read(method + '/' + method + '.txt', format='ascii')
data['RA'].name = 'ra'
data['DEC'].name = 'dec'
data.write(method + '.fits')
else:
kwargs = {}
if 'panstarrs' in method:
kwargs['rastr'] = 'ramean'
kwargs['decstr'] = 'decmean'
data = merge_cat(method, **kwargs)
if o['mode'] == 'test':
image_root += '_shift'
method += '-test'
report('Running PyBDSM on LOFAR image, please wait...')
catfile = image_root + '.offset_cat.fits'
gaulfile = catfile.replace('cat', 'gaul')
if os.path.isfile(catfile):
warn('Catalogue already exists, skipping pybdsf run')
else:
if o['mode'] == 'test':
suffix = 'facetRestored'
else:
suffix = 'restored'
pbimage = image_root + '.int.' + suffix + '.fits'
nonpbimage = image_root + '.app.' + suffix + '.fits'
img = bdsm.process_image(pbimage,
detection_image=nonpbimage,
thresh_isl=4.0,
thresh_pix=5.0,
rms_box=(150, 15),
rms_map=True,
mean_map='zero',
ini_method='intensity',
adaptive_rms_box=True,
adaptive_thresh=150,
rms_box_bright=(60, 15),
group_by_isl=False,
group_tol=10.0,
output_opts=True,
output_all=True,
atrous_do=False,
flagging_opts=True,
flag_maxsize_fwhm=0.5,
advanced_opts=True,
blank_limit=None)
img.write_catalog(outfile=catfile,
catalog_type='srl',
format='fits',
correct_proj='True')
img.write_catalog(outfile=gaulfile,
catalog_type='gaul',
format='fits',
correct_proj='True')
lofar = Table.read(catfile)
print len(lofar), 'LOFAR sources before filtering'
filter = (lofar['E_RA'] * 3600.0) < 2.0
filter &= (lofar['E_DEC'] * 3600.0) < 2.0
filter &= (lofar['Maj'] * 3600.0) < 10
lofar = lofar[filter]
print len(lofar), 'LOFAR sources after filtering'
regfile = image_root + '.tessel.reg'
cra, cdec = get_centpos()
report('Set up structure')
NDir = np.load("image_dirin_SSD_m.npy.ClusterCat.npy").shape[0]
oo = Offsets(method,
n=NDir,
imroot=image_root,
cellsize=o['cellsize'],
fitmethod=o['fit'])
report('Label table')
lofar_l = oo.r.add_facet_labels(lofar)
report('Finding offsets')
oo.find_offsets(lofar_l, data)
report('Fitting offsets')
oo.fit_offsets()
report('Making plots and saving output')
#oo.plot_fits(method+'-fits.pdf')
oo.save_fits()
oo.plot_offsets()
if 'test' not in o['mode']:
oo.save(method + '-fit_state.pickle')
report('Making astrometry error map, please wait')
oo.make_astrometry_map('astromap.fits', 20)
oo.offsets_to_facetshift('facet-offset.txt')
print 'Blanking the mosaic...'
blank_mosaic('mosaic.fits',himsize)
if args.do_lowres:
print 'Making the low-resolution mosaic...'
header,himsize=make_header(maxsep,mospointingname,pointingdict[mospointingname][1],pointingdict[mospointingname][2],4.5,20.0)
mos_args.header=header
mos_args.rootname='low'
mos_args.do_lowres=True
mos_args.astromap_blank=False # don't bother with low-res map
#with open('low-mosaic-header.pickle','w') as f:
# pickle.dump(header,f)
make_mosaic(mos_args)
print 'Blanking the mosaic...'
blank_mosaic('low-mosaic.fits',himsize)
print 'Now running PyBDSF to extract sources'
catprefix='mosaic'
img = bdsm.process_image('mosaic-blanked.fits', thresh_isl=4.0, thresh_pix=5.0, rms_box=(160,50), rms_map=True, mean_map='zero', ini_method='intensity', adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(60,15), group_by_isl=False, group_tol=10.0,output_opts=True, output_all=True, atrous_do=True,atrous_jmax=4, flagging_opts=True, flag_maxsize_fwhm=0.5,advanced_opts=True, blank_limit=None,frequency=restfrq)
img.write_catalog(outfile=catprefix +'.cat.fits',catalog_type='srl',format='fits',correct_proj='True')
img.export_image(outfile=catprefix +'.rms.fits',img_type='rms',img_format='fits',clobber=True)
img.export_image(outfile=catprefix +'.resid.fits',img_type='gaus_resid',img_format='fits',clobber=True)
img.export_image(outfile=catprefix +'.pybdsmmask.fits',img_type='island_mask',img_format='fits',clobber=True)
img.write_catalog(outfile=catprefix +'.cat.reg',catalog_type='srl',format='ds9',correct_proj='True')
def find_sources(infile): img = bdsf.process_image(infile,thresh_pix=12.0,thresh_isl=5.0) img.write_catalog(format='ascii',catalog_type='gaul',clobber=True,incl_empty=True)
def do_sourcefinding(imagename):
img = bdsf.process_image(imagename, adaptive_rms_box=True, advanced_opts=True,\
atrous_do=False, psf_vary_do=True, psf_snrcut=5.0, psf_snrcutstack=10.0,\
output_opts=True, output_all=True, opdir_overwrite='append', beam=(4.1666e-4, 4.1666e-4, 0),\
blank_limit=None, thresh='hard', thresh_isl=5.0, thresh_pix=7.0, psf_snrtop=0.30)
return img
def main(image_name, mask_name, atrous_do=False, threshisl=0.0, threshpix=0.0, rmsbox=None,
rmsbox_bright=(35, 7), iterate_threshold=False, adaptive_rmsbox=False, img_format='fits',
threshold_format='float', trim_by=0.0, vertices_file=None, atrous_jmax=6,
pad_to_size=None, skip_source_detection=False, region_file=None, nsig=1.0,
reference_ra_deg=None, reference_dec_deg=None, cellsize_deg=0.000417,
use_adaptive_threshold=False, make_blank_image=False, adaptive_thresh=150.0,
exclude_cal_region=False, dilate=0):
"""
Make a clean mask and return clean threshold
Parameters
----------
image_name : str
Filename of input image from which mask will be made. If the image does
not exist or make_blank_image is True, a template image with center at
(reference_ra_deg, reference_dec_deg) will be made internally
mask_name : str
Filename of output mask image
atrous_do : bool, optional
Use wavelet module of PyBDSF?
threshisl : float, optional
Value of thresh_isl PyBDSF parameter
threshpix : float, optional
Value of thresh_pix PyBDSF parameter
rmsbox : tuple of floats, optional
Value of rms_box PyBDSF parameter
rmsbox_bright : tuple of floats, optional
Value of rms_box_bright PyBDSF parameter
iterate_threshold : bool, optional
If True, threshold will be lower in 20% steps until
at least one island is found
adaptive_rmsbox : tuple of floats, optional
Value of adaptive_rms_box PyBDSF parameter
img_format : str, optional
Format of output mask image (one of 'fits' or 'casa')
threshold_format : str, optional
Format of output threshold (one of 'float' or 'str_with_units')
trim_by : float, optional
Fraction by which the perimeter of the output mask will be
trimmed (zeroed)
vertices_file : str, optional
Filename of file with vertices (must be a pickle file containing
a dictionary with the vertices in the 'vertices' entry)
atrous_jmax : int, optional
Value of atrous_jmax PyBDSF parameter
pad_to_size : int, optional
Pad output mask image to a size of pad_to_size x pad_to_size
skip_source_detection : bool, optional
If True, source detection is not run on the input image
region_file : str, optional
Filename of region file in CASA format to use as the mask. If update_user_mask
is True, regions in region_file are unioned with ones found by the
source finder
nsig : float, optional
Number of sigma of returned threshold value
reference_ra_deg : float, optional
RA for center of output mask image
reference_dec_deg : float, optional
Dec for center of output mask image
cellsize_deg : float, optional
Size of a pixel in degrees
use_adaptive_threshold : bool, optional
If True, use an adaptive threshold estimated from the negative values in
the image
make_blank_image : bool, optional
If True, a blank template image is made. In this case, reference_ra_deg
and reference_dec_deg must be specified
adaptive_thresh : float, optional
If adaptive_rmsbox is True, this value sets the threshold above
which a source will use the small rms box
exclude_cal_region : bool, optional
If True, and a vertices_file is given, the calibrator region is also
exclude from the output mask
dilate : int, optional
Number of dilation iterations for PyBDSF mask
Returns
-------
result : dict
Dict with nsig-sigma rms threshold
"""
if rmsbox is not None and type(rmsbox) is str:
rmsbox = eval(rmsbox)
if type(rmsbox_bright) is str:
rmsbox_bright = eval(rmsbox_bright)
if pad_to_size is not None and type(pad_to_size) is str:
pad_to_size = int(pad_to_size)
if type(atrous_do) is str:
if atrous_do.lower() == 'true':
atrous_do = True
threshisl = 4.0 # override user setting to ensure proper source fitting
else:
atrous_do = False
if type(iterate_threshold) is str:
if iterate_threshold.lower() == 'true':
iterate_threshold = True
else:
iterate_threshold = False
if type(adaptive_rmsbox) is str:
if adaptive_rmsbox.lower() == 'true':
adaptive_rmsbox = True
else:
adaptive_rmsbox = False
if type(skip_source_detection) is str:
if skip_source_detection.lower() == 'true':
skip_source_detection = True
else:
skip_source_detection = False
if type(use_adaptive_threshold) is str:
if use_adaptive_threshold.lower() == 'true':
use_adaptive_threshold = True
else:
use_adaptive_threshold = False
if reference_ra_deg is not None and reference_dec_deg is not None:
reference_ra_deg = float(reference_ra_deg)
reference_dec_deg = float(reference_dec_deg)
if type(make_blank_image) is str:
if make_blank_image.lower() == 'true':
make_blank_image = True
else:
make_blank_image = False
if not os.path.exists(image_name):
make_blank_image = True
if type(exclude_cal_region) is str:
if exclude_cal_region.lower() == 'true':
exclude_cal_region = True
else:
exclude_cal_region = False
dilate = int(dilate)
if make_blank_image:
print('Making empty template image...')
if not skip_source_detection:
print('ERROR: Source detection cannot be done on an empty image')
sys.exit(1)
if reference_ra_deg is not None and reference_dec_deg is not None:
image_name = mask_name + '.tmp'
make_template_image(image_name, reference_ra_deg, reference_dec_deg,
cellsize_deg=float(cellsize_deg))
else:
print('ERROR: a reference position must be given to make an empty template image')
sys.exit(1)
trim_by = float(trim_by)
atrous_jmax = int(atrous_jmax)
threshpix = float(threshpix)
threshisl = float(threshisl)
nsig = float(nsig)
adaptive_thresh = float(adaptive_thresh)
threshold = 0.0
nisl = 0
if not skip_source_detection:
if vertices_file is not None:
# Modify the input image to blank the regions outside of the polygon
blank_image.main(image_name, vertices_file, image_name+'.blanked',
blank_value='nan')
image_name += '.blanked'
if use_adaptive_threshold:
# Get an estimate of the rms
img = bdsf.process_image(image_name, mean_map='zero', rms_box=rmsbox,
thresh_pix=threshpix, thresh_isl=threshisl,
atrous_do=atrous_do, thresh='hard',
adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=adaptive_thresh,
rms_box_bright=rmsbox_bright, rms_map=True, quiet=True,
atrous_jmax=atrous_jmax, stop_at='isl')
# Find min and max pixels
max_neg_val = abs(np.min(img.ch0_arr))
max_neg_pos = np.where(img.ch0_arr == np.min(img.ch0_arr))
max_pos_val = abs(np.max(img.ch0_arr))
max_pos_pos = np.where(img.ch0_arr == np.max(img.ch0_arr))
# Estimate new thresh_isl from min pixel value's sigma, but don't let
# it get higher than 1/2 of the peak's sigma
threshisl_neg = 2.0 * max_neg_val / img.rms_arr[max_neg_pos][0]
max_sigma = max_pos_val / img.rms_arr[max_pos_pos][0]
if threshisl_neg > max_sigma / 2.0:
threshisl_neg = max_sigma / 2.0
# Use the new threshold only if it is larger than the user-specified one
if threshisl_neg > threshisl:
threshisl = threshisl_neg
if not atrous_do:
stop_at = 'isl'
else:
stop_at = None
if iterate_threshold:
# Start with given threshold and lower it until we get at least one island
nisl = 0
while nisl == 0:
img = bdsf.process_image(image_name, mean_map='zero', rms_box=rmsbox,
thresh_pix=threshpix, thresh_isl=threshisl,
atrous_do=atrous_do, thresh='hard',
adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=adaptive_thresh,
rms_box_bright=rmsbox_bright, rms_map=True, quiet=True,
atrous_jmax=atrous_jmax, stop_at=stop_at)
nisl = img.nisl
threshpix /= 1.2
threshisl /= 1.2
if threshpix < 5.0:
break
else:
img = bdsf.process_image(image_name, mean_map='zero', rms_box=rmsbox,
thresh_pix=threshpix, thresh_isl=threshisl,
atrous_do=atrous_do, thresh='hard',
adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=adaptive_thresh,
rms_box_bright=rmsbox_bright, rms_map=True, quiet=True,
atrous_jmax=atrous_jmax, stop_at=stop_at)
nisl = img.nisl
if nisl == 0:
if region_file is None or region_file == '[]':
print('No islands found. Clean mask cannot be made.')
return {'threshold_5sig': 'None'}
else:
# Continue on and use user-supplied region file
threshold = nsig * img.clipped_rms
else:
# Write out the mask
img.export_image(img_type='island_mask', mask_dilation=dilate, outfile=mask_name,
img_format=img_format, clobber=True)
# Check if there are large islands present (indicating that multi-scale
# clean is needed)
has_large_isl = False
for isl in img.islands:
if isl.size_active > 100:
# Assuming normal sampling, a size of 100 pixels would imply
# a source of ~ 10 beams
has_large_isl = True
if (vertices_file is not None or trim_by > 0 or pad_to_size is not None
or (region_file is not None and region_file != '[]')
or skip_source_detection):
# Alter the mask in various ways
if skip_source_detection or nisl == 0:
# Read the image
mask_im = pim.image(image_name)
else:
# Read the PyBDSF mask
mask_im = pim.image(mask_name)
data = mask_im.getdata()
coordsys = mask_im.coordinates()
if reference_ra_deg is not None and reference_dec_deg is not None:
values = coordsys.get_referencevalue()
values[2][0] = reference_dec_deg/180.0*np.pi
values[2][1] = reference_ra_deg/180.0*np.pi
coordsys.set_referencevalue(values)
imshape = mask_im.shape()
del(mask_im)
if pad_to_size is not None:
imsize = pad_to_size
coordsys['direction'].set_referencepixel([imsize/2, imsize/2])
pixmin = (imsize - imshape[2]) / 2
if pixmin < 0:
print("The padded size must be larger than the original size.")
sys.exit(1)
pixmax = pixmin + imshape[2]
data_pad = np.zeros((1, 1, imsize, imsize), dtype=np.float32)
data_pad[0, 0, pixmin:pixmax, pixmin:pixmax] = data[0, 0]
new_mask = pim.image('', shape=(1, 1, imsize, imsize), coordsys=coordsys)
new_mask.putdata(data_pad)
else:
new_mask = pim.image('', shape=imshape, coordsys=coordsys)
new_mask.putdata(data)
data = new_mask.getdata()
if skip_source_detection or nisl == 0:
if region_file is not None and region_file != '[]':
# Unmask all pixels. We will fill the masked regions
# below
data[:] = 0
else:
# Mask all pixels
data[:] = 1
if region_file is not None and region_file != '[]':
# Merge the CASA regions with the mask
casa_polys = read_casa_polys(region_file.strip('[]"'), new_mask)
for poly in casa_polys:
# Find unmasked regions
unmasked_ind = np.where(data[0, 0] == 0)
# Find distance to nearest poly edge and mask those that
# are inside the casa region (dist > 0)
dist = poly.is_inside(unmasked_ind[0], unmasked_ind[1])
inside_ind = np.where(dist > 0.0)
if len(inside_ind[0]) > 0:
data[0, 0, unmasked_ind[0][inside_ind], unmasked_ind[1][inside_ind]] = 1
if vertices_file is not None:
# Modify the clean mask to exclude regions outside of the polygon
vertices = read_vertices(vertices_file)
poly = vertices_to_poly(vertices, new_mask)
if exclude_cal_region:
cal_vertices = read_vertices(vertices_file, cal_only=True)
cal_poly = vertices_to_poly(cal_vertices, new_mask)
# Find masked regions
masked_ind = np.where(data[0, 0])
# Find distance to nearest poly edge and unmask those that
# are outside the facet (dist < 0) and inside the calibrator region
# (cal_dist > 0)
dist = poly.is_inside(masked_ind[0], masked_ind[1])
outside_ind = np.where(dist < 0.0)
if len(outside_ind[0]) > 0:
data[0, 0, masked_ind[0][outside_ind], masked_ind[1][outside_ind]] = 0
if exclude_cal_region:
masked_ind = np.where(data[0, 0])
cal_dist = cal_poly.is_inside(masked_ind[0], masked_ind[1])
inside_ind = np.where(cal_dist > 0.0)
if len(inside_ind[0]) > 0:
data[0, 0, masked_ind[0][inside_ind], masked_ind[1][inside_ind]] = 0
if trim_by > 0.0:
sh = np.shape(data)
margin = int(sh[2] * trim_by / 2.0 )
data[0, 0, 0:sh[2], 0:margin] = 0
data[0, 0, 0:margin, 0:sh[3]] = 0
data[0, 0, 0:sh[2], sh[3]-margin:sh[3]] = 0
data[0, 0, sh[2]-margin:sh[2], 0:sh[3]] = 0
# Save changes
new_mask.putdata(data)
if img_format == 'fits':
new_mask.tofits(mask_name, overwrite=True)
elif img_format == 'casa':
new_mask.saveas(mask_name, overwrite=True)
else:
print('Output image format "{}" not understood.'.format(img_format))
sys.exit(1)
if not skip_source_detection:
if threshold_format == 'float':
return {'threshold_5sig': nsig * img.clipped_rms, 'multiscale': has_large_isl}
elif threshold_format == 'str_with_units':
# This is done to get around the need for quotes around strings in casapy scripts
# 'casastr/' is removed by the generic pipeline
return {'threshold_5sig': 'casastr/{0}Jy'.format(nsig * img.clipped_rms),
'multiscale': has_large_isl}
else:
return {'threshold_5sig': '0.0'}
