def check_rmsmap(self, img, rms):
"""Calculates the statistics of the rms map and decides, when
rms_map=None, whether to take the map (if variance
is significant) or a constant value
"""
from math import sqrt
mylog = mylogger.logging.getLogger("PyBDSM." + img.log +
"Rmsimage.Checkrms ")
cdelt = img.wcs_obj.acdelt[:2]
bm = (img.beam[0], img.beam[1])
fw_pix = sqrt(N.product(bm) / abs(N.product(cdelt)))
if img.masked:
unmasked = N.where(~img.mask_arr)
stdsub = N.std(rms[unmasked])
maxrms = N.max(rms[unmasked])
else:
stdsub = N.std(rms)
maxrms = N.max(rms)
rms_expect = img.clipped_rms / sqrt(2) / img.rms_box[0] * fw_pix
mylog.debug(
'%s %10.6f %s' %
('Standard deviation of rms image = ', stdsub * 1000.0, 'mJy'))
mylog.debug(
'%s %10.6f %s' %
('Expected standard deviation = ', rms_expect * 1000.0, 'mJy'))
if stdsub > 1.1 * rms_expect:
img.use_rms_map = True
mylogger.userinfo(mylog, 'Variation in rms image significant')
else:
img.use_rms_map = False
mylogger.userinfo(mylog, 'Variation in rms image not significant')
return img
def run_dist_com_all(self, task, wdir, command, SBs=[], nodes='', group=False):
"""Run a run_dist_com on all nodes and all SBs unless specified a restriction.
If no restriction are specified, run the command for each node and each SB.
"""
s = self.get_status()
if s == None:
self.mylog.error("Not connected to a cluster.")
return False
njobs = 0
for node in s:
# shall we proceed with this node?
if (node in nodes or nodes == ''):
if group:
nodeSBs = s[node]['group']
else:
nodeSBs = s[node]['sb']
for SB in nodeSBs:
# shall we proceed with this SB?
if (int(SB) in SBs or SBs == []):
rcommand = command.replace('$SB', SB)
rwdir = wdir.replace('$SB', SB)
# for groups rename the variable
if group: SB = 'g'+SB
self.run_dist_com(task, rwdir, rcommand, SB, node)
njobs += 1
mylogger.userinfo(self.mylog, "Launched "+str(njobs)+" jobs.")
def check_meanmap(self, img, mean):
"""Calculates the statistics of the mean map and decides, when
mean_map=None, whether to take the map (if variance
is significant) or a constant value
"""
from math import sqrt
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Rmsimage.Checkmean ")
cdelt = img.wcs_obj.acdelt[:2]
bm = (img.beam[0], img.beam[1])
fw_pix = sqrt(N.product(bm)/abs(N.product(cdelt)))
if img.masked:
unmasked = N.where(~img.mask_arr)
stdsub = N.std(mean[unmasked])
maxmean = N.max(mean[unmasked])
else:
stdsub = N.std(mean)
maxmean = N.max(mean)
rms_expect = img.clipped_rms/img.rms_box[0]*fw_pix
mylog.debug('%s %10.6f %s' % ('Standard deviation of mean image = ', stdsub*1000.0, 'mJy'))
mylog.debug('%s %10.6f %s' % ('Expected standard deviation = ', rms_expect*1000.0, 'mJy'))
# For mean map, use a higher threshold than for the rms map, as radio images
# should rarely, if ever, have significant variations in the mean
if stdsub > 5.0*rms_expect:
img.mean_map_type = 'map'
mylogger.userinfo(mylog, 'Variation in mean image significant')
else:
if img.confused:
img.mean_map_type = 'zero'
else:
img.mean_map_type = 'const'
mylogger.userinfo(mylog, 'Variation in mean image not significant')
return img
def check_rmsmap(self, img, rms):
"""Calculates the statistics of the rms map and decides, when
rms_map=None, whether to take the map (if variance
is significant) or a constant value
"""
from math import sqrt
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Rmsimage.Checkrms ")
cdelt = img.wcs_obj.acdelt[:2]
bm = (img.beam[0], img.beam[1])
fw_pix = sqrt(N.product(bm)/abs(N.product(cdelt)))
if img.masked:
unmasked = N.where(~img.mask_arr)
stdsub = N.std(rms[unmasked])
maxrms = N.max(rms[unmasked])
else:
stdsub = N.std(rms)
maxrms = N.max(rms)
rms_expect = img.clipped_rms/sqrt(2)/img.rms_box[0]*fw_pix
mylog.debug('%s %10.6f %s' % ('Standard deviation of rms image = ', stdsub*1000.0, 'mJy'))
mylog.debug('%s %10.6f %s' % ('Expected standard deviation = ', rms_expect*1000.0, 'mJy'))
if stdsub > 1.1*rms_expect:
img.use_rms_map = True
mylogger.userinfo(mylog, 'Variation in rms image significant')
else:
img.use_rms_map = False
mylogger.userinfo(mylog, 'Variation in rms image not significant')
return img
def check_meanmap(self, img, mean):
"""Calculates the statistics of the mean map and decides, when
mean_map=None, whether to take the map (if variance
is significant) or a constant value
"""
from math import sqrt
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Rmsimage.Checkmean ")
cdelt = img.wcs_obj.acdelt[:2]
bm = (img.beam[0], img.beam[1])
fw_pix = sqrt(N.product(bm)/abs(N.product(cdelt)))
if img.masked:
unmasked = N.where(~img.mask_arr)
stdsub = N.std(mean[unmasked])
maxmean = N.max(mean[unmasked])
else:
stdsub = N.std(mean)
maxmean = N.max(mean)
rms_expect = img.clipped_rms/img.rms_box[0]*fw_pix
mylog.debug('%s %10.6f %s' % ('Standard deviation of mean image = ', stdsub*1000.0, 'mJy'))
mylog.debug('%s %10.6f %s' % ('Expected standard deviation = ', rms_expect*1000.0, 'mJy'))
# For mean map, use a higher threshold than for the rms map, as radio images
# should rarely, if ever, have significant variations in the mean
if stdsub > 5.0*rms_expect:
img.mean_map_type = 'map'
mylogger.userinfo(mylog, 'Variation in mean image significant')
else:
if img.confused:
img.mean_map_type = 'zero'
else:
img.mean_map_type = 'const'
mylogger.userinfo(mylog, 'Variation in mean image not significant')
return img
def run_dist_com_all(self,
task,
wdir,
command,
SBs=[],
nodes='',
group=False):
"""Run a run_dist_com on all nodes and all SBs unless specified a restriction.
If no restriction are specified, run the command for each node and each SB.
"""
s = self.get_status()
if s == None:
self.mylog.error("Not connected to a cluster.")
return False
njobs = 0
for node in s:
# shall we proceed with this node?
if (node in nodes or nodes == ''):
if group:
nodeSBs = s[node]['group']
else:
nodeSBs = s[node]['sb']
for SB in nodeSBs:
# shall we proceed with this SB?
if (int(SB) in SBs or SBs == []):
rcommand = command.replace('$SB', SB)
rwdir = wdir.replace('$SB', SB)
# for groups rename the variable
if group: SB = 'g' + SB
self.run_dist_com(task, rwdir, rcommand, SB, node)
njobs += 1
mylogger.userinfo(self.mylog, "Launched " + str(njobs) + " jobs.")
def set_client(self):
"""Set current cluster client"""
from IPython.parallel import Client
try:
self._cluster_client = Client(profile='ssh')
mylogger.userinfo(self.mylog, "Connected to a cluster.")
return True
except:
return False
def set_client(self):
"""Set current cluster client"""
from IPython.parallel import Client
try:
self._cluster_client = Client(profile='ssh')
mylogger.userinfo(self.mylog, "Connected to a cluster.")
return True
except:
return False
def calculate_maps(self, img, data, mean, rms, mask, map_opts, do_adapt,
bright_pt_coords=[], rms_box2=None,
logname=None, ncores=None):
"""Calls map_2d and checks for problems"""
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Rmsimage.Calcmaps ")
rms_ok = False
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Rmsimage.Calcmaps ")
opts = img.opts
kappa = map_opts[0]
while not rms_ok:
self.map_2d(data, mean, rms, mask, *map_opts, do_adapt=do_adapt,
bright_pt_coords=bright_pt_coords, rms_box2=rms_box2,
logname=logname, ncores=ncores)
if N.any(rms < 0.0):
rms_ok = False
if (opts.rms_box_bright is None and do_adapt) or (opts.rms_box is None and not do_adapt):
# Increase box by 20%
if do_adapt:
new_width = int(img.rms_box_bright[0]*1.2)
if new_width == img.rms_box_bright[0]:
new_width = img.rms_box_bright[0] + 1
new_step = int(new_width/3.0)
img.rms_box_bright = (new_width, new_step)
if img.rms_box_bright[0] > min(img.ch0_arr.shape)/4.0:
mylogger.userinfo(mylog, 'Size of rms_box_bright larger than 1/4 of image size')
mylogger.userinfo(mylog, 'Using constant background rms and mean')
img.use_rms_map = False
img.rms_box = img.rms_box_bright
img.mean_map_type = 'const'
rms_ok = True
else:
map_opts = (kappa, img.rms_box_bright, opts.spline_rank)
else:
new_width = int(img.rms_box[0]*1.2)
if new_width == img.rms_box[0]:
new_width = img.rms_box[0] + 1
new_step = int(new_width/3.0)
img.rms_box = (new_width, new_step)
if img.rms_box[0] > min(img.ch0_arr.shape)/4.0:
mylogger.userinfo(mylog, 'Size of rms_box larger than 1/4 of image size')
mylogger.userinfo(mylog, 'Using constant background rms and mean')
img.use_rms_map = False
img.mean_map_type = 'const'
rms_ok = True
else:
map_opts = (kappa, img.rms_box, opts.spline_rank)
else:
# User has specified box size, use order=1 to prevent negatives
if opts.spline_rank > 1:
mylog.warning('Negative values found in rms map interpolated with spline_rank = %i' % opts.spline_rank)
mylog.warning('Using spline_rank = 1 (bilinear interpolation) instead')
if do_adapt:
map_opts = (kappa, img.rms_box_bright, 1)
else:
map_opts = (kappa, img.rms_box, 1)
else:
rms_ok = True
return mean, rms
def run_dist_com(self, task, wdir, command, SB='', node=''):
""" Run a distributed command using information on the _cluster_status.
The node parameter is required, the SB not (used only for job identification
and to schedule a job after already submitted jobs on that SB are done).
"""
s = self.get_status()
if s == None:
self.mylog.error("Not connected to a cluster.")
return False
if node == '':
self.mylog.error("Node parameter required to run a command.")
return False
# define the function called by engines
@interactive
def f(c, node='', SB='', task='', wdir=''):
import os, subprocess
if wdir != '' and os.path.isdir(wdir): os.chdir(wdir)
s = subprocess.Popen(c, shell=True,\
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
s.wait()
(out, err) = s.communicate()
return {'node':node, 'SB':SB, 'task':task, 'command':c, \
'out':out, 'err':err}
mylogger.userinfo(
self.mylog, "Launching on node: " + node + " (SB: " + SB +
", wdir:" + wdir + ")\n" + command)
try:
scheduler = self.get_client().load_balanced_view(s[node]['e'])
# Exec a job on a SB only if previous jobs on that
# SB are finished and finished without errors
time.sleep(0.1) # TODO: check if this minimize the buffer problem
after_ids = self.getSBids(SB)
with scheduler.temp_flags(after=after_ids, retries=3):
scheduler.apply_async(f,
command,
node=node,
SB=SB,
task=task,
wdir=wdir)
except Exception, e:
self.mylog.exception("Cannot launch the command: " + command + \
"(Task: " + task + " - SB: " + SB + " - Node: " + node + ")")
def run_dist_com(self, task, wdir, command, SB='', node=''):
""" Run a distributed command using information on the _cluster_status.
The node parameter is required, the SB not (used only for job identification
and to schedule a job after already submitted jobs on that SB are done).
"""
s = self.get_status()
if s == None:
self.mylog.error("Not connected to a cluster.")
return False
if node == '':
self.mylog.error("Node parameter required to run a command.")
return False
# define the function called by engines
@interactive
def f(c, node='', SB='', task='', wdir=''):
import os, subprocess
if wdir != '' and os.path.isdir(wdir): os.chdir(wdir)
s = subprocess.Popen(c, shell=True,\
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
s.wait()
(out, err) = s.communicate()
return {'node':node, 'SB':SB, 'task':task, 'command':c, \
'out':out, 'err':err}
mylogger.userinfo(self.mylog,"Launching on node: "+node+" (SB: "+SB+", wdir:"+wdir+")\n"+command)
try:
scheduler = self.get_client().load_balanced_view(s[node]['e'])
# Exec a job on a SB only if previous jobs on that
# SB are finished and finished without errors
time.sleep(0.1) # TODO: check if this minimize the buffer problem
after_ids = self.getSBids(SB)
with scheduler.temp_flags(after=after_ids, retries=3):
scheduler.apply_async(f, command, node=node, SB=SB, task=task, wdir=wdir)
except Exception, e:
self.mylog.exception("Cannot launch the command: " + command + \
"(Task: " + task + " - SB: " + SB + " - Node: " + node + ")")
def output_rmsbox_size(self, img):
"""Prints rms/mean box size"""
opts = img.opts
do_adapt = opts.adaptive_rms_box
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"RMSimage")
if (opts.rms_map is not False) or (opts.mean_map not in ['zero', 'const']):
if do_adapt:
if opts.rms_box_bright is None:
mylogger.userinfo(mylog, 'Derived rms_box (box size, step size)',
'(' + str(img.rms_box_bright[0]) + ', ' +
str(img.rms_box_bright[1]) + ') pixels (small scale)')
else:
mylogger.userinfo(mylog, 'Using user-specified rms_box',
'(' + str(img.rms_box_bright[0]) + ', ' +
str(img.rms_box_bright[1]) + ') pixels (small scale)')
if opts.rms_box is None:
mylogger.userinfo(mylog, 'Derived rms_box (box size, step size)',
'(' + str(img.rms_box[0]) + ', ' +
str(img.rms_box[1]) + ') pixels (large scale)')
else:
mylogger.userinfo(mylog, 'Using user-specified rms_box',
'(' + str(img.rms_box[0]) + ', ' +
str(img.rms_box[1]) + ') pixels (large scale)')
else:
if opts.rms_box is None:
mylogger.userinfo(mylog, 'Derived rms_box (box size, step size)',
'(' + str(img.rms_box[0]) + ', ' +
str(img.rms_box[1]) + ') pixels')
else:
mylogger.userinfo(mylog, 'Using user-specified rms_box',
'(' + str(img.rms_box[0]) + ', ' +
str(img.rms_box[1]) + ') pixels')
def __call__(self, img):
# for each island, get the gaussians into a list and then send them to process
# src_index is source number, starting from 0
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Gaul2Srl")
mylogger.userinfo(mylog, 'Grouping Gaussians into sources')
img.aperture = img.opts.aperture
if img.aperture is not None and img.aperture <= 0.0:
mylog.warn('Specified aperture is <= 0. Skipping aperture fluxes.')
img.aperture = None
src_index = -1
dsrc_index = 0
sources = []
dsources = []
no_gaus_islands = []
for iisl, isl in enumerate(img.islands):
isl_sources = []
isl_dsources = []
g_list = []
for g in isl.gaul:
if g.flag == 0:
g_list.append(g)
if len(g_list) > 0:
if len(g_list) == 1:
src_index, source = self.process_single_gaussian(img, g_list, src_index, code = 'S')
sources.append(source)
isl_sources.append(source)
else:
src_index, source = self.process_CM(img, g_list, isl, src_index)
sources.extend(source)
isl_sources.extend(source)
else:
if not img.waveletimage:
dg = isl.dgaul[0]
no_gaus_islands.append((isl.island_id, dg.centre_pix[0], dg.centre_pix[1]))
# Put in the dummy Source as the source and use negative IDs
g_list = isl.dgaul
dsrc_index, dsource = self.process_single_gaussian(img, g_list, dsrc_index, code = 'S')
dsources.append(dsource)
isl_dsources.append(dsource)
isl.sources = isl_sources
isl.dsources = isl_dsources
img.sources = sources
img.dsources = dsources
img.nsrc = src_index + 1
mylogger.userinfo(mylog, "Number of sources formed from Gaussians",
str(img.nsrc))
if not img.waveletimage and not img._pi and len(no_gaus_islands) > 0 and not img.opts.quiet:
message = 'All Gaussians were flagged for the following island'
if len(no_gaus_islands) == 1:
message += ':\n'
else:
message += 's:\n'
for isl_id in no_gaus_islands:
message += ' Island #%i (x=%i, y=%i)\n' % isl_id
if len(no_gaus_islands) == 1:
message += 'Please check this island. If it is a valid island and\n'
else:
message += 'Please check these islands. If they are valid islands and\n'
if img.opts.atrous_do:
message += 'should be fit, try adjusting the flagging options (use\n'\
'show_fit with "ch0_flagged=True" to see the flagged Gaussians).'
else:
message += 'should be fit, try adjusting the flagging options (use\n'\
'show_fit with "ch0_flagged=True" to see the flagged Gaussians)\n'\
'or enabling the wavelet module (with "atrous_do=True").'
message += '\nTo include empty islands in output source catalogs, set\n'\
'incl_empty=True in the write_catalog task.'
mylog.warning(message)
img.completed_Ops.append('gaul2srl')
def __call__(self, img):
mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Wavelet")
if img.opts.atrous_do:
if img.nisl == 0:
mylog.warning("No islands found. Skipping wavelet decomposition.")
img.completed_Ops.append('wavelet_atrous')
return
mylog.info("Decomposing gaussian residual image into a-trous wavelets")
bdir = img.basedir + '/wavelet/'
if img.opts.output_all:
if not os.path.isdir(bdir): os.makedirs(bdir)
if not os.path.isdir(bdir + '/residual/'): os.makedirs(bdir + '/residual/')
if not os.path.isdir(bdir + '/model/'): os.makedirs(bdir + '/model/')
dobdsm = img.opts.atrous_bdsm_do
filter = {'tr':{'size':3, 'vec':[1. / 4, 1. / 2, 1. / 4], 'name':'Triangle'},
'b3':{'size':5, 'vec':[1. / 16, 1. / 4, 3. / 8, 1. / 4, 1. / 16], 'name':'B3 spline'}}
if dobdsm: wchain, wopts = self.setpara_bdsm(img)
n, m = img.ch0_arr.shape
# Calculate residual image that results from normal (non-wavelet) Gaussian fitting
Op_make_residimage()(img)
resid = img.resid_gaus_arr
lpf = img.opts.atrous_lpf
if lpf not in ['b3', 'tr']: lpf = 'b3'
jmax = img.opts.atrous_jmax
l = len(filter[lpf]['vec']) # 1st 3 is arbit and 2nd 3 is whats expected for a-trous
if jmax < 1 or jmax > 15: # determine jmax
# Check if largest island size is
# smaller than 1/3 of image size. If so, use it to determine jmax.
min_size = min(resid.shape)
max_isl_shape = (0, 0)
for isl in img.islands:
if isl.image.shape[0] * isl.image.shape[1] > max_isl_shape[0] * max_isl_shape[1]:
max_isl_shape = isl.image.shape
if max_isl_shape != (0, 0) and min(max_isl_shape) < min(resid.shape) / 3.0:
min_size = min(max_isl_shape) * 4.0
else:
min_size = min(resid.shape)
jmax = int(floor(log((min_size / 3.0 * 3.0 - l) / (l - 1) + 1) / log(2.0) + 1.0)) + 1
if min_size * 0.55 <= (l + (l - 1) * (2 ** (jmax) - 1)): jmax = jmax - 1
img.wavelet_lpf = lpf
img.wavelet_jmax = jmax
mylog.info("Using " + filter[lpf]['name'] + ' filter with J_max = ' + str(jmax))
img.atrous_islands = []
img.atrous_gaussians = []
img.atrous_sources = []
img.atrous_opts = []
img.resid_wavelets_arr = cp(img.resid_gaus_arr)
im_old = img.resid_wavelets_arr
total_flux = 0.0
ntot_wvgaus = 0
stop_wav = False
pix_masked = N.where(N.isnan(resid) == True)
jmin = 1
if img.opts.ncores is None:
numcores = 1
else:
numcores = img.opts.ncores
for j in range(jmin, jmax + 1): # extra +1 is so we can do bdsm on cJ as well
mylogger.userinfo(mylog, "\nWavelet scale #" + str(j))
im_new = self.atrous(im_old, filter[lpf]['vec'], lpf, j, numcores=numcores, use_scipy_fft=img.opts.use_scipy_fft)
im_new[pix_masked] = N.nan # since fftconvolve wont work with blanked pixels
if img.opts.atrous_sum:
w = im_new
else:
w = im_old - im_new
im_old = im_new
suffix = 'w' + `j`
filename = img.imagename + '.atrous.' + suffix + '.fits'
if img.opts.output_all:
func.write_image_to_file('fits', filename, w, img, bdir)
mylog.info('%s %s' % ('Wrote ', img.imagename + '.atrous.' + suffix + '.fits'))
# now do bdsm on each wavelet image.
if dobdsm:
wopts['filename'] = filename
wopts['basedir'] = bdir
box = img.rms_box[0]
y1 = (l + (l - 1) * (2 ** (j - 1) - 1))
bs = max(5 * y1, box) # changed from 10 to 5
if bs > min(n, m) / 2:
wopts['rms_map'] = False
wopts['mean_map'] = 'const'
wopts['rms_box'] = None
else:
wopts['rms_box'] = (bs, bs/3)
if hasattr(img, '_adapt_rms_isl_pos'):
bs_bright = max(5 * y1, img.rms_box_bright[0])
if bs_bright < bs/1.5:
wopts['adaptive_rms_box'] = True
wopts['rms_box_bright'] = (bs_bright, bs_bright/3)
else:
wopts['adaptive_rms_box'] = False
if j <= 3:
wopts['ini_gausfit'] = 'default'
else:
wopts['ini_gausfit'] = 'nobeam'
wid = (l + (l - 1) * (2 ** (j - 1) - 1))# / 3.0
b1, b2 = img.pixel_beam()[0:2]
b1 = b1 * fwsig
b2 = b2 * fwsig
cdelt = img.wcs_obj.acdelt[:2]
wimg = Image(wopts)
wimg.beam = (sqrt(wid * wid + b1 * b1) * cdelt[0] * 2.0, sqrt(wid * wid + b2 * b2) * cdelt[1] * 2.0, 0.0)
wimg.orig_beam = img.beam
wimg.pixel_beam = img.pixel_beam
wimg.pixel_beamarea = img.pixel_beamarea
wimg.log = 'Wavelet.'
wimg.basedir = img.basedir
wimg.extraparams['bbsprefix'] = suffix
wimg.extraparams['bbsname'] = img.imagename + '.wavelet'
wimg.extraparams['bbsappend'] = True
wimg.bbspatchnum = img.bbspatchnum
wimg.waveletimage = True
wimg.j = j
if hasattr(img, '_adapt_rms_isl_pos'):
wimg._adapt_rms_isl_pos = img._adapt_rms_isl_pos
self.init_image_simple(wimg, img, w, '.atrous.' + suffix)
for op in wchain:
op(wimg)
gc.collect()
if isinstance(op, Op_islands) and img.opts.atrous_orig_isl:
if wimg.nisl > 0:
# Find islands that do not share any pixels with
# islands in original ch0 image.
good_isl = []
# Make original rank image boolean; rank counts from 0, with -1 being
# outside any island
orig_rankim_bool = N.array(img.pyrank + 1, dtype = bool)
# Multiply rank images
old_islands = orig_rankim_bool * (wimg.pyrank + 1) - 1
# Exclude islands that don't overlap with a ch0 island.
valid_ids = set(old_islands.flatten())
for idx, wvisl in enumerate(wimg.islands):
if idx in valid_ids:
wvisl.valid = True
good_isl.append(wvisl)
else:
wvisl.valid = False
wimg.islands = good_isl
wimg.nisl = len(good_isl)
mylogger.userinfo(mylog, "Number of islands found", '%i' %
wimg.nisl)
# Renumber islands:
for wvindx, wvisl in enumerate(wimg.islands):
wvisl.island_id = wvindx
if isinstance(op, Op_gausfit):
# If opts.atrous_orig_isl then exclude Gaussians outside of
# the original ch0 islands
nwvgaus = 0
if img.opts.atrous_orig_isl:
gaul = wimg.gaussians
tot_flux = 0.0
if img.ngaus == 0:
gaus_id = -1
else:
gaus_id = img.gaussians[-1].gaus_num
wvgaul = []
for g in gaul:
if not hasattr(g, 'valid'):
g.valid = False
if not g.valid:
try:
isl_id = img.pyrank[int(g.centre_pix[0] + 1), int(g.centre_pix[1] + 1)]
except IndexError:
isl_id = -1
if isl_id >= 0:
isl = img.islands[isl_id]
gcenter = (g.centre_pix[0] - isl.origin[0],
g.centre_pix[1] - isl.origin[1])
if not isl.mask_active[gcenter]:
gaus_id += 1
gcp = Gaussian(img, g.parameters[:], isl.island_id, gaus_id)
gcp.gaus_num = gaus_id
gcp.wisland_id = g.island_id
gcp.jlevel = j
g.valid = True
isl.gaul.append(gcp)
isl.ngaus += 1
img.gaussians.append(gcp)
nwvgaus += 1
tot_flux += gcp.total_flux
else:
g.valid = False
g.jlevel = 0
else:
g.valid = False
g.jlevel = 0
vg = []
for g in wimg.gaussians:
if g.valid:
vg.append(g)
wimg.gaussians = vg
mylogger.userinfo(mylog, "Number of valid wavelet Gaussians", str(nwvgaus))
else:
# Keep all Gaussians and merge islands that overlap
tot_flux = check_islands_for_overlap(img, wimg)
# Now renumber the islands and adjust the rank image before going to next wavelet image
renumber_islands(img)
total_flux += tot_flux
if img.opts.interactive and has_pl:
dc = '\033[34;1m'
nc = '\033[0m'
print dc + '--> Displaying islands and rms image...' + nc
if max(wimg.ch0_arr.shape) > 4096:
print dc + '--> Image is large. Showing islands only.' + nc
wimg.show_fit(rms_image=False, mean_image=False, ch0_image=False,
ch0_islands=True, gresid_image=False, sresid_image=False,
gmodel_image=False, smodel_image=False, pyramid_srcs=False)
else:
wimg.show_fit()
prompt = dc + "Press enter to continue or 'q' stop fitting wavelet images : " + nc
answ = raw_input_no_history(prompt)
while answ != '':
if answ == 'q':
img.wavelet_jmax = j
stop_wav = True
break
answ = raw_input_no_history(prompt)
if len(wimg.gaussians) > 0:
img.resid_wavelets_arr = self.subtract_wvgaus(img.opts, img.resid_wavelets_arr, wimg.gaussians, wimg.islands)
if img.opts.atrous_sum:
im_old = self.subtract_wvgaus(img.opts, im_old, wimg.gaussians, wimg.islands)
if stop_wav == True:
break
pyrank = N.zeros(img.pyrank.shape, dtype=N.int32)
for i, isl in enumerate(img.islands):
isl.island_id = i
for g in isl.gaul:
g.island_id = i
for dg in isl.dgaul:
dg.island_id = i
pyrank[isl.bbox] += N.invert(isl.mask_active) * (i + 1)
pyrank -= 1 # align pyrank values with island ids and set regions outside of islands to -1
img.pyrank = pyrank
pdir = img.basedir + '/misc/'
img.ngaus += ntot_wvgaus
img.total_flux_gaus += total_flux
mylogger.userinfo(mylog, "Total flux density in model on all scales" , '%.3f Jy' % img.total_flux_gaus)
if img.opts.output_all:
func.write_image_to_file('fits', img.imagename + '.atrous.cJ.fits',
im_new, img, bdir)
mylog.info('%s %s' % ('Wrote ', img.imagename + '.atrous.cJ.fits'))
func.write_image_to_file('fits', img.imagename + '.resid_wavelets.fits',
(img.ch0_arr - img.resid_gaus_arr + img.resid_wavelets_arr), img, bdir + '/residual/')
mylog.info('%s %s' % ('Wrote ', img.imagename + '.resid_wavelets.fits'))
func.write_image_to_file('fits', img.imagename + '.model_wavelets.fits',
(img.resid_gaus_arr - img.resid_wavelets_arr), img, bdir + '/model/')
mylog.info('%s %s' % ('Wrote ', img.imagename + '.model_wavelets.fits'))
img.completed_Ops.append('wavelet_atrous')
def __call__(self, img):
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Islands")
opts = img.opts
minsize = opts.minpix_isl
if minsize == None:
minsize = int(img.pixel_beamarea()/3.0) # 1/3 of beam area in pixels
if minsize < 6:
minsize = 6 # Need at least 6 pixels to obtain good fits
mylogger.userinfo(mylog, "Minimum number of pixels per island", '%i' %
minsize)
img.minpix_isl = minsize
if opts.detection_image != '':
# Use a different image for island detection. The detection
# image and the measurement image must have the same shape
# and be registered. Otherwise, one could reproject the
# detection image using, e.g., the Kapteyn package.
#
# First, set up up an Image object and run a limited
# op_chain.
from . import _run_op_list
mylogger.userinfo(mylog, "\nDetermining islands from detection image")
det_chain, det_opts = self.setpara_bdsm(img, opts.detection_image)
det_img = Image(det_opts)
det_img.log = 'Detection image'
success = _run_op_list(det_img, det_chain)
if not success:
return
# Check that the ch0 images are the same size
ch0_map = img.ch0_arr
det_ch0_map = det_img.ch0_arr
det_shape = det_ch0_map.shape
ch0_shape = ch0_map.shape
if det_shape != ch0_shape:
raise RuntimeError("Detection image shape does not match that of input image.")
# Run through islands and correct the image and rms, mean and max values
img.island_labels = det_img.island_labels
corr_islands = []
mean_map = img.mean_arr
rms_map = img.rms_arr
for i, isl in enumerate(det_img.islands):
islcp = isl.copy(img.pixel_beamarea(), image=ch0_map[isl.bbox], mean=mean_map[isl.bbox], rms=rms_map[isl.bbox])
islcp.island_id = i
corr_islands.append(islcp)
img.islands = corr_islands
img.nisl = len(img.islands)
img.pyrank = det_img.pyrank
img.minpix_isl = det_img.minpix_isl
mylogger.userinfo(mylog, "\nContinuing processing using primary image")
else:
if opts.src_ra_dec != None:
mylogger.userinfo(mylog, "Constructing islands at user-supplied source locations")
img.islands = self.coords_to_isl(img, opts)
else:
img.islands = self.ndimage_alg(img, opts)
img.nisl = len(img.islands)
mylogger.userinfo(mylog, "Number of islands found", '%i' %
len(img.islands))
ch0_map = img.ch0_arr
ch0_shape = ch0_map.shape
pyrank = N.zeros(ch0_shape, dtype=N.int32) - 1
for i, isl in enumerate(img.islands):
isl.island_id = i
if i == 0:
pyrank[isl.bbox] = N.invert(isl.mask_active) - 1
else:
pyrank[isl.bbox] = N.invert(isl.mask_active) * i - isl.mask_active
if opts.output_all: write_islands(img)
if opts.savefits_rankim:
func.write_image_to_file(img.use_io, img.imagename + '_pyrank.fits', pyrank, img)
img.pyrank = pyrank
img.completed_Ops.append('islands')
return img
def __call__(self, img):
from time import time
import functions as func
import itertools
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Gausfit")
if len(img.islands) == 0:
img.gaussians = []
img.ngaus = 0
img.total_flux_gaus = 0.0
img.completed_Ops.append('gausfit')
return img
bar = statusbar.StatusBar('Fitting islands with Gaussians .......... : ',
0, img.nisl)
opts = img.opts
if opts.quiet == False and opts.verbose_fitting == False:
bar.start()
iter_ngmax = 10
min_maxsize = 50.0
maxsize = opts.splitisl_maxsize
min_peak_size = 30.0
peak_size = opts.peak_maxsize
if maxsize < min_maxsize:
maxsize = min_maxsize
opts.splitisl_maxsize = min_maxsize
if peak_size < min_peak_size:
peak_size = min_peak_size
opts.peak_maxsize = min_peak_size
# Set up multiproccessing. First create a simple copy of the Image
# object that contains the minimal data needed.
opts_dict = opts.to_dict()
img_simple = Image(opts_dict)
img_simple.pixel_beamarea = img.pixel_beamarea
img_simple.pixel_beam = img.pixel_beam
img_simple.thresh_pix = img.thresh_pix
img_simple.minpix_isl = img.minpix_isl
img_simple.clipped_mean = img.clipped_mean
img_simple.beam2pix = img.beam2pix
img_simple.beam = img.beam
# Next, define the weights to use when distributing islands among cores.
# The weight should scale with the processing time. At the moment
# we use the island area, but other parameters may be better.
weights = []
for isl in img.islands:
weights.append(isl.size_active)
# Now call the parallel mapping function. Returns a list of [gaul, fgaul]
# for each island.
gaus_list = mp.parallel_map(func.eval_func_tuple,
itertools.izip(itertools.repeat(self.process_island),
img.islands, itertools.repeat(img_simple),
itertools.repeat(opts)), numcores=opts.ncores,
bar=bar, weights=weights)
for isl in img.islands:
### now convert gaussians into Gaussian objects and store
idx = isl.island_id
gaul = gaus_list[idx][0]
fgaul = gaus_list[idx][1]
dgaul = []
gaul = [Gaussian(img, par, idx, gidx)
for (gidx, par) in enumerate(gaul)]
if len(gaul) == 0:
# No good Gaussians were fit. In this case, make a dummy
# Gaussian located at the island center so
# that the source may still be included in output catalogs.
# These dummy Gaussians all have an ID of -1. They do not
# appear in any of the source or island Gaussian lists except
# the island dgaul list.
if opts.src_ra_dec != None:
# Center the dummy Gaussian on the user-specified source position
posn_isl = (isl.shape[0]/2.0, isl.shape[1]/2.0)
posn_img = (isl.shape[0]/2.0 + isl.origin[0], isl.shape[1]/2.0 + isl.origin[1])
par = [isl.image[posn_isl], posn_img[0], posn_img[1], 0.0, 0.0, 0.0]
else:
# Center the dummy Gaussian on the maximum pixel
posn = N.unravel_index(N.argmax(isl.image*~isl.mask_active), isl.shape) + N.array(isl.origin)
par = [isl.max_value, posn[0], posn[1], 0.0, 0.0, 0.0]
dgaul = [Gaussian(img, par, idx, -1)]
gidx = 0
fgaul= [Gaussian(img, par, idx, gidx + gidx2 + 1, flag)
for (gidx2, (flag, par)) in enumerate(fgaul)]
isl.gaul = gaul
isl.fgaul= fgaul
isl.dgaul = dgaul
gaussian_list = [g for isl in img.islands for g in isl.gaul]
img.gaussians = gaussian_list
### put in the serial number of the gaussians for the whole image
n = 0
nn = 0
tot_flux = 0.0
if img.waveletimage:
# store the wavelet scale for each Gaussian
# (wavelet img's have a img.j attribute)
j = img.j
else:
j = 0
for isl in img.islands:
m = 0
for g in isl.gaul:
n += 1; m += 1
g.gaus_num = n - 1
tot_flux += g.total_flux
for dg in isl.dgaul:
nn -= 1
dg.gaus_num = nn
isl.ngaus = m
img.ngaus = n
img.total_flux_gaus = tot_flux
mylogger.userinfo(mylog, "Total number of Gaussians fit to image",
str(n))
if not img._pi and not img.waveletimage:
mylogger.userinfo(mylog, "Total flux density in model", '%.3f Jy' %
tot_flux)
# Check if model flux is very different from sum of flux in image
if img.ch0_sum_jy > 0 and not img._pi:
if img.total_flux_gaus/img.ch0_sum_jy < 0.5 or \
img.total_flux_gaus/img.ch0_sum_jy > 2.0:
mylog.warn('Total flux density in model is %0.2f times sum of pixels '\
'in input image. Large residuals may remain.' %
(img.total_flux_gaus/img.ch0_sum_jy,))
# Check if there are many Gaussians with deconvolved size of 0 in one
# axis but not in the other. Don't bother to do this for wavelet images.
fraction_1d = self.check_for_1d_gaussians(img)
if fraction_1d > 0.5 and img.beam != None and img.waveletimage == False:
mylog.warn('After deconvolution, more than 50% of Gaussians are '\
"1-D. Unless you're fitting an extended source, "\
"beam may be incorrect.")
img.completed_Ops.append('gausfit')
return img
img, op_chain = get_op_chain(img)
if op_chain is not None:
_run_op_list(img, op_chain)
img._prev_opts = img.opts.to_dict()
return True
except RuntimeError, err:
# Catch and log error
mylog.error(str(err))
# Re-throw error if the user is not in the interactive shell
if img._is_interactive_shell:
return False
else:
raise
except KeyboardInterrupt:
mylogger.userinfo(mylog, "\n\033[31;1mAborted\033[0m")
return False
def get_op_chain(img):
"""Determines the optimal Op chain for an Image object.
This is useful when reprocessing an Image object. For example,
if Gaussians were already fit, but the user now wants to use
shapelets, we do not need to re-run Op_gausfit, etc.
Note that any new options added to opts.py should also be
added here. If not, a full reprocessing will be done if the
new option is changed.
"""
from . import default_chain
def __call__(self, img):
# for each island, get the gaussians into a list and then send them to process
# src_index is source number, starting from 0
mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Gaul2Srl")
mylogger.userinfo(mylog, 'Grouping Gaussians into sources')
img.aperture = img.opts.aperture
if img.aperture is not None and img.aperture <= 0.0:
mylog.warn('Specified aperture is <= 0. Skipping aperture fluxes.')
img.aperture = None
src_index = -1
dsrc_index = 0
sources = []
dsources = []
no_gaus_islands = []
for iisl, isl in enumerate(img.islands):
isl_sources = []
isl_dsources = []
g_list = []
for g in isl.gaul:
if g.flag == 0:
g_list.append(g)
if len(g_list) > 0:
if len(g_list) == 1:
src_index, source = self.process_single_gaussian(img,
g_list,
src_index,
code='S')
sources.append(source)
isl_sources.append(source)
else:
src_index, source = self.process_CM(
img, g_list, isl, src_index)
sources.extend(source)
isl_sources.extend(source)
else:
if not img.waveletimage:
dg = isl.dgaul[0]
no_gaus_islands.append(
(isl.island_id, dg.centre_pix[0], dg.centre_pix[1]))
# Put in the dummy Source as the source and use negative IDs
g_list = isl.dgaul
dsrc_index, dsource = self.process_single_gaussian(
img, g_list, dsrc_index, code='S')
dsources.append(dsource)
isl_dsources.append(dsource)
isl.sources = isl_sources
isl.dsources = isl_dsources
img.sources = sources
img.dsources = dsources
img.nsrc = src_index + 1
mylogger.userinfo(mylog, "Number of sources formed from Gaussians",
str(img.nsrc))
if not img.waveletimage and not img._pi and len(
no_gaus_islands) > 0 and not img.opts.quiet:
message = 'All Gaussians were flagged for the following island'
if len(no_gaus_islands) == 1:
message += ':\n'
else:
message += 's:\n'
for isl_id in no_gaus_islands:
message += ' Island #%i (x=%i, y=%i)\n' % isl_id
if len(no_gaus_islands) == 1:
message += 'Please check this island. If it is a valid island and\n'
else:
message += 'Please check these islands. If they are valid islands and\n'
if img.opts.atrous_do:
message += 'should be fit, try adjusting the flagging options (use\n'\
'show_fit with "ch0_flagged=True" to see the flagged Gaussians).'
else:
message += 'should be fit, try adjusting the flagging options (use\n'\
'show_fit with "ch0_flagged=True" to see the flagged Gaussians)\n'\
'or enabling the wavelet module (with "atrous_do=True").'
message += '\nTo include empty islands in output source catalogs, set\n'\
'incl_empty=True in the write_catalog task.'
mylog.warning(message)
img.completed_Ops.append('gaul2srl')
def __call__(self, img):
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Preprocess")
bstat = func.bstat
if img.opts.kappa_clip is None:
kappa = -img.pixel_beamarea()
else:
kappa = img.opts.kappa_clip
if img.opts.polarisation_do:
pols = ['I', 'Q', 'U', 'V']
ch0images = [img.ch0_arr, img.ch0_Q_arr, img.ch0_U_arr, img.ch0_V_arr]
img.clipped_mean_QUV = []
img.clipped_rms_QUV = []
else:
pols = ['I'] # assume I is always present
ch0images = [img.ch0_arr]
if hasattr(img, 'rms_mask'):
mask = img.rms_mask
else:
mask = img.mask_arr
opts = img.opts
for ipol, pol in enumerate(pols):
image = ch0images[ipol]
### basic stats
mean, rms, cmean, crms, cnt = bstat(image, mask, kappa)
if cnt > 198: cmean = mean; crms = rms
if pol == 'I':
if func.approx_equal(crms, 0.0, rel=None):
raise RuntimeError('Clipped rms appears to be zero. Check for regions '\
'with values of 0 and\nblank them (with NaNs) '\
'or use trim_box to exclude them.')
img.raw_mean = mean
img.raw_rms = rms
img.clipped_mean= cmean
img.clipped_rms = crms
mylog.info('%s %.4f %s %.4f %s ' % ("Raw mean (Stokes I) = ", mean*1000.0, \
'mJy and raw rms = ',rms*1000.0, 'mJy'))
mylog.info('%s %.4f %s %s %.4f %s ' % ("sigma clipped mean (Stokes I) = ", cmean*1000.0, \
'mJy and ','sigma clipped rms = ',crms*1000.0, 'mJy'))
else:
img.clipped_mean_QUV.append(cmean)
img.clipped_rms_QUV.append(crms)
mylog.info('%s %s %s %.4f %s %s %.4f %s ' % ("sigma clipped mean (Stokes ", pol, ") = ", cmean*1000.0, \
'mJy and ','sigma clipped rms = ',crms*1000.0, 'mJy'))
image = img.ch0_arr
# Check if pixels are outside the universe
if opts.check_outsideuniv:
mylogger.userinfo(mylog, "Checking for pixels outside the universe")
noutside_univ = self.outside_univ(img)
img.noutside_univ = noutside_univ
frac_blank = round(float(noutside_univ)/float(image.shape[0]*image.shape[1]),3)
mylogger.userinfo(mylog, "Number of additional pixels blanked", str(noutside_univ)
+' ('+str(frac_blank*100.0)+'%)')
else:
noutside_univ = 0
# If needed, (re)mask the image
if noutside_univ > 0:
mask = N.isnan(img.ch0_arr)
masked = mask.any()
img.masked = masked
if masked:
img.mask_arr = mask
img.blankpix = N.sum(mask)
### max/min pixel value & coordinates
shape = image.shape[0:2]
if mask != None:
img.blankpix = N.sum(mask)
if img.blankpix == 0:
max_idx = image.argmax()
min_idx = image.argmin()
else:
max_idx = N.nanargmax(image)
min_idx = N.nanargmin(image)
img.maxpix_coord = N.unravel_index(max_idx, shape)
img.minpix_coord = N.unravel_index(min_idx, shape)
img.max_value = image.flat[max_idx]
img.min_value = image.flat[min_idx]
### Solid angle of the image
cdelt = N.array(img.wcs_obj.acdelt[:2])
img.omega = N.product(shape)*abs(N.product(cdelt))/(180.*180./pi/pi)
### Total flux in ch0 image
if 'atrous' in img.filename or img._pi or img.log == 'Detection image':
# Don't do this estimate for atrous wavelet images
# or polarized intensity image,
# as it doesn't give the correct flux. Also, ignore
# the flux in the detection image, as it's likely
# wrong (e.g., not corrected for the primary beam).
img.ch0_sum_jy = 0
else:
im_flux = N.nansum(image)/img.pixel_beamarea() # Jy
img.ch0_sum_jy = im_flux
mylogger.userinfo(mylog, 'Flux from sum of (non-blank) pixels',
'%.3f Jy' % (im_flux,))
### if image seems confused, then take background mean as zero instead
alpha_sourcecounts = 2.5 # approx diff src count slope. 2.2?
if opts.bmpersrc_th is None:
n = (image >= 5.*crms).sum()
if n <= 0:
n = 1
mylog.info('No pixels in image > 5-sigma.')
mylog.info('Taking number of pixels above 5-sigma as 1.')
img.bmpersrc_th = N.product(shape)/((alpha_sourcecounts-1.)*n)
mylog.info('%s %6.2f' % ('Estimated bmpersrc_th = ', img.bmpersrc_th))
else:
img.bmpersrc_th = opts.bmpersrc_th
mylog.info('%s %6.2f' % ('Taking default bmpersrc_th = ', img.bmpersrc_th))
confused = False
if opts.mean_map == 'default':
if opts.bmpersrc_th <= 25. or cmean/crms >= 0.1:
confused = True
img.confused = confused
mylog.info('Parameter confused is '+str(img.confused))
img.completed_Ops.append('preprocess')
return img
def __call__(self, img):
mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Collapse")
if img.opts.polarisation_do:
pols = ["I", "Q", "U", "V"] # make sure I is done first
else:
pols = ["I"] # assume I is always present
img.ch0_Q_arr = None
img.ch0_U_arr = None
img.ch0_V_arr = None
if img.shape[1] > 1:
c_mode = img.opts.collapse_mode
chan0 = img.opts.collapse_ch0
c_list = img.opts.collapse_av
c_wts = img.opts.collapse_wt
if c_list == []:
c_list = N.arange(img.shape[1])
if len(c_list) == 1:
c_mode = "single"
chan0 = c_list[0]
img.collapse_ch0 = chan0
ch0sh = img.image_arr.shape[2:]
if img.opts.polarisation_do:
ch0images = ["ch0_arr", "ch0_Q_arr", "ch0_U_arr", "ch0_V_arr"]
else:
ch0images = ["ch0_arr"]
# assume all Stokes images have the same blank pixels as I:
blank = N.isnan(img.image_arr[0])
hasblanks = blank.any()
if img.opts.kappa_clip is None:
kappa = -img.pixel_beamarea()
else:
kappa = img.opts.kappa_clip
mean, rms, cmean, crms = chan_stats(img, kappa)
img.channel_mean = mean
img.channel_rms = rms
img.channel_clippedmean = cmean
img.channel_clippedrms = crms
for ipol, pol in enumerate(pols):
if c_mode == "single":
if pol == "I":
ch0 = img.image_arr[0, chan0]
img.ch0_arr = ch0
mylogger.userinfo(
mylog,
"Source extraction will be " "done on channel",
"%i (%.3f MHz)" % (chan0, img.frequency / 1e6),
)
else:
ch0[:] = img.image_arr[ipol, chan0][:]
img.__setattr__(ch0images[ipol][:], ch0)
if c_mode == "average":
if not hasblanks:
if pol == "I":
ch0, wtarr = avspc_direct(c_list, img.image_arr[0], img.channel_clippedrms, c_wts)
else:
# use wtarr from the I image, which is always collapsed first
ch0, wtarr = avspc_direct(
c_list, img.image_arr[ipol], img.channel_clippedrms, c_wts, wtarr=wtarr
)
else:
if pol == "I":
ch0, wtarr = avspc_blanks(c_list, img.image_arr[0], img.channel_clippedrms, c_wts)
else:
# use wtarr from the I image, which is always collapsed first
ch0, wtarr = avspc_blanks(
c_list, img.image_arr[ipol], img.channel_clippedrms, c_wts, wtarr=wtarr
)
img.__setattr__(ch0images[ipol][:], ch0)
if pol == "I":
img.avspc_wtarr = wtarr
init_freq_collapse(img, wtarr)
if c_wts == "unity":
mylogger.userinfo(mylog, "Channels averaged with " "uniform weights")
else:
mylogger.userinfo(mylog, "Channels averaged with " "weights=(1/rms)^2")
mylogger.userinfo(mylog, "Source extraction will be " 'done on averaged ("ch0") image')
mylogger.userinfo(mylog, "Frequency of averaged " "image", "%.3f MHz" % (img.frequency / 1e6,))
str1 = " ".join(str(n) for n in c_list)
mylog.debug("%s %s" % ("Channels averaged : ", str1))
str1 = " ".join(["%9.4e" % n for n in wtarr])
mylog.debug("%s %s %s" % ("Channel weights : ", str1, '; unity=zero if c_wts="rms"'))
if img.opts.output_all:
func.write_image_to_file(img.use_io, img.imagename + ".ch0_" + pol + ".fits", ch0, img)
mylog.debug("%s %s " % ("Writing file ", img.imagename + ".ch0_" + pol + ".fits"))
else:
# Only one channel in image
image = img.image_arr
img.ch0_arr = image[0, 0]
mylogger.userinfo(mylog, "Frequency of image", "%.3f MHz" % (img.frequency / 1e6,))
if img.opts.polarisation_do:
for pol in pols[1:]:
if pol == "Q":
img.ch0_Q_arr = image[1, 0][:]
if pol == "U":
img.ch0_U_arr = image[2, 0][:]
if pol == "V":
img.ch0_V_arr = image[3, 0][:]
# create mask if needed (assume all pols have the same mask as I)
image = img.ch0_arr
mask = N.isnan(image)
img.blankpix = N.sum(mask)
frac_blank = round(float(img.blankpix) / float(image.shape[0] * image.shape[1]), 3)
mylogger.userinfo(mylog, "Number of blank pixels", str(img.blankpix) + " (" + str(frac_blank * 100.0) + "%)")
# Check whether the input image might be an AWimage. If so, and there
# are no blank pixels, tell the user that they might to set blank_limit.
# Once the AWimager incorporates blanking, this check can be removed.
if img.opts.blank_limit is None and (
img.blankpix == 0
and ("restored" in img.filename.lower() or "corr" in img.filename.lower() or "aw" in img.filename.lower())
):
check_low = True
else:
check_low = False
if img.opts.blank_limit is not None or check_low:
import scipy
import sys
if check_low:
threshold = 1e-5
else:
threshold = img.opts.blank_limit
mylogger.userinfo(mylog, "Blanking pixels with values " "below %.1e Jy/beam" % (threshold,))
bad = abs(image) < threshold
original_stdout = sys.stdout # keep a reference to STDOUT
sys.stdout = func.NullDevice() # redirect the real STDOUT
count = scipy.signal.convolve2d(bad, N.ones((3, 3)), mode="same")
sys.stdout = original_stdout # turn STDOUT back on
mask_low = count >= 5
if check_low:
nlow = len(N.where(mask_low)[0])
if nlow / float(image.shape[0] * image.shape[1]) > 0.2:
mylog.warn(
"A significant area of the image has very low values. To blank\nthese regions (e.g., because they are outside the primary beam), set the\nblank_limit option (values of 1e-5 to 1e-4 Jy/beam usually work well).\n"
)
else:
image[N.where(mask_low)] = N.nan
mask = N.isnan(image)
img.blankpix = N.sum(mask)
frac_blank = round(float(img.blankpix) / float(image.shape[0] * image.shape[1]), 3)
mylogger.userinfo(
mylog, "Total number of blanked pixels", str(img.blankpix) + " (" + str(frac_blank * 100.0) + "%)"
)
masked = mask.any()
img.masked = masked
if masked:
img.mask_arr = mask
else:
img.mask_arr = None
if img.blankpix == image.shape[0] * image.shape[1]:
# ALL pixels are blanked!
raise RuntimeError("All pixels in the image are blanked.")
img.completed_Ops.append("collapse")
def init_beam(self, img):
"""Initialize beam parameters, and conversion routines
to convert beam to/from pixel coordinates"""
from const import fwsig
mylog = mylogger.logging.getLogger("PyBDSM.InitBeam")
hdr = img.header
cdelt1, cdelt2 = img.wcs_obj.acdelt[0:2]
### define beam conversion routines:
def beam2pix(x):
""" Converts beam in deg to pixels. Use when no dependence on
position is appropriate.
Input beam angle should be degrees CCW from North at image center.
The output beam angle is degrees CCW from the +y axis of the image.
"""
bmaj, bmin, bpa = x
s1 = abs(bmaj / cdelt1)
s2 = abs(bmin / cdelt2)
th = bpa
return (s1, s2, th)
def pix2beam(x):
""" Converts beam in pixels to deg. Use when no dependence on
position is appropriate.
Input beam angle should be degrees CCW from the +y axis of the image.
The output beam angle is degrees CCW from North at image center.
"""
s1, s2, th = x
bmaj = abs(s1 * cdelt1)
bmin = abs(s2 * cdelt2)
bpa = th
if bmaj < bmin:
bmaj, bmin = bmin, bmaj
bpa += 90.0
bpa = divmod(bpa, 180)[1] ### bpa lies between 0 and 180
return [bmaj, bmin, bpa]
def pixel_beam():
"""Returns the beam in sigma units in pixels"""
pbeam = beam2pix(img.beam)
return (pbeam[0]/fwsig, pbeam[1]/fwsig, pbeam[2])
def pixel_beamarea():
"""Returns the beam area in pixels"""
pbeam = beam2pix(img.beam)
return 1.1331 * pbeam[0] * pbeam[1]
### Get the beam information from the header
found = False
if img.opts.beam is not None:
beam = img.opts.beam
else:
try:
beam = (hdr['BMAJ'], hdr['BMIN'], hdr['BPA'])
found = True
except:
### try see if AIPS as put the beam in HISTORY as usual
for h in hdr.get_history():
# Check if h is a string or a FITS Card object (long headers are
# split into Cards as of PyFITS 3.0.4)
if not isinstance(h, str):
hstr = h.value
else:
hstr = h
if N.all(['BMAJ' in hstr, 'BMIN' in hstr, 'BPA' in hstr, 'CLEAN' in hstr]):
try:
dum, dum, dum, bmaj, dum, bmin, dum, bpa = hstr.split()
except ValueError:
try:
dum, dum, bmaj, dum, bmin, dum, bpa, dum, dum = hstr.split()
except ValueError:
break
beam = (float(bmaj), float(bmin), float(bpa))
found = True
if not found: raise RuntimeError("No beam information found in image header.")
### convert beam into pixels (at image center)
pbeam = beam2pix(beam)
pbeam = (pbeam[0] / fwsig, pbeam[1] / fwsig, pbeam[2]) # IN SIGMA UNITS
### and store it
img.pix2beam = pix2beam
img.beam2pix = beam2pix
img.beam = beam # FWHM size in degrees
img.pixel_beam = pixel_beam # IN SIGMA UNITS in pixels
img.pixel_beamarea = pixel_beamarea
mylogger.userinfo(mylog, 'Beam shape (major, minor, pos angle)',
'(%.5e, %.5e, %s) degrees' % (beam[0], beam[1],
round(beam[2], 1)))
def __call__(self, img):
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"RMSimage")
mylogger.userinfo(mylog, "Calculating background rms and mean images")
if img.opts.polarisation_do:
pols = ['I', 'Q', 'U', 'V']
ch0_images = [img.ch0_arr, img.ch0_Q_arr, img.ch0_U_arr, img.ch0_V_arr]
cmeans = [img.clipped_mean] + img.clipped_mean_QUV
crmss = [img.clipped_rms] + img.clipped_rms_QUV
else:
pols = ['I'] # assume I is always present
ch0_images = [img.ch0_arr]
cmeans = [img.clipped_mean]
crmss = [img.clipped_rms]
mask = img.mask_arr
opts = img.opts
cdelt = N.array(img.wcs_obj.acdelt[:2])
# Determine box size for rms/mean map calculations.
# If user specifies rms_box, use it. Otherwise, use either an
# adaptive binning scheme that shrinks the box near
# the brightest sources or estimate rms_box from bright sources.
#
# The adaptive scheme calculates the rms/mean map
# at two different scales:
# 1) using a large rms_box, set by size of largest source
# 2) using a small rms_box, set by size of largest bright source
# Then, the rms and mean values at a given point are determined
# by a weighted average of the values in the maps at the two
# scales.
fwsig = const.fwsig
min_adapt_threshold = 10.0
if opts.adaptive_thresh == None:
adapt_thresh = 50.0
start_thresh = 500.0
else:
adapt_thresh = opts.adaptive_thresh
if adapt_thresh < min_adapt_threshold:
adapt_thresh = min_adapt_threshold
opts.adaptive_thresh = min_adapt_threshold
start_thresh = adapt_thresh
brightsize = None
isl_pos = []
do_adapt = img.opts.adaptive_rms_box
img.use_rms_map = None
img.mean_map_type = None
# 'size' of brightest source
kappa1 = 3.0
try:
brightsize = int(round(2.*img.beam[0]/cdelt[0]/fwsig*
sqrt(2.*log(img.max_value/(kappa1*crms)))))
except:
brightsize = int(round(2.*img.beam[0]/cdelt[0]/fwsig))
mylog.info('Estimated size of brightest source (pixels) = '+str(brightsize))
# Using clipped mean and rms and a starting threshold of 500 sigma,
# search for bright sources. If fewer than 5 are found, reduce
# threshold until limit set by adapt_thresh is hit.
cmean = cmeans[0]
crms = crmss[0]
image = ch0_images[0]
shape = image.shape
isl_size_bright = []
isl_area_highthresh = []
isl_peak = []
max_isl_brightsize = 0.0
threshold = start_thresh
if do_adapt:
mylogger.userinfo(mylog, "Using adaptive scaling of rms_box")
while len(isl_size_bright) < 5 and threshold >= adapt_thresh:
isl_size_bright=[]
isl_maxposn = []
act_pixels = (image-cmean)/threshold >= crms
threshold *= 0.8
if isinstance(mask, N.ndarray):
act_pixels[mask] = False
rank = len(image.shape)
connectivity = nd.generate_binary_structure(rank, rank)
labels, count = nd.label(act_pixels, connectivity)
slices = nd.find_objects(labels)
for idx, s in enumerate(slices):
isl_size_bright.append(max([s[0].stop-s[0].start, s[1].stop-s[1].start]))
size_area = (labels[s] == idx+1).sum()/img.pixel_beamarea()*2.0
isl_area_highthresh.append(size_area)
isl_maxposn.append(tuple(N.array(N.unravel_index(N.argmax(image[s]), image[s].shape))+\
N.array((s[0].start, s[1].start))))
isl_peak.append(nd.maximum(image[s], labels[s], idx+1))
# Check islands found above at thresh_isl threshold to determine if
# the bright source is embedded inside a large island or not. If it is,
# exclude it from the bright-island list. Also find the size of the
# largest island at this threshold to set the large-scale rms_box
bright_threshold = threshold
threshold = 10.0
act_pixels = (image-cmean)/threshold >= crms
if isinstance(mask, N.ndarray):
act_pixels[mask] = False
rank = len(image.shape)
connectivity = nd.generate_binary_structure(rank, rank)
labels, count = nd.label(act_pixels, connectivity)
slices = nd.find_objects(labels)
isl_size = []
isl_size_highthresh = []
isl_size_lowthresh = []
isl_snr = []
thratio = threshold/bright_threshold
for idx, s in enumerate(slices):
isl_area_lowthresh = (labels[s] == idx+1).sum()/img.pixel_beamarea()*2.0
isl_maxposn_lowthresh = tuple(N.array(N.unravel_index(N.argmax(image[s]), image[s].shape))+
N.array((s[0].start, s[1].start)))
isl_size += [s[0].stop-s[0].start, s[1].stop-s[1].start]
if do_adapt and isl_maxposn_lowthresh in isl_maxposn:
bright_indx = isl_maxposn.index(isl_maxposn_lowthresh)
if isl_area_lowthresh < 25.0 or isl_area_lowthresh/isl_area_highthresh[bright_indx] < 8.0:
isl_pos.append(isl_maxposn_lowthresh)
isl_size_lowthresh.append(max([s[0].stop-s[0].start, s[1].stop-s[1].start]))
isl_size_highthresh.append(isl_size_bright[bright_indx])
isl_snr.append(isl_peak[bright_indx]/crms)
if len(isl_size) == 0:
max_isl_size = 0.0
else:
max_isl_size = max(isl_size)
mylog.info('Maximum extent of largest 10-sigma island using clipped rms (pixels) = '+str(max_isl_size))
if len(isl_size_highthresh) == 0:
max_isl_size_highthresh = 0.0
max_isl_size_lowthresh = 0.0
else:
max_isl_size_highthresh = max(isl_size_highthresh)
max_isl_size_lowthresh = max(isl_size_lowthresh)
avg_max_isl_size = (max_isl_size_highthresh + max_isl_size_lowthresh) / 2.0
if hasattr(img, '_adapt_rms_isl_pos'):
isl_pos = img._adapt_rms_isl_pos # set isl_pos to existing value (for wavelet analysis)
if len(isl_pos) == 0:
# No bright sources found
do_adapt = False
else:
img._adapt_rms_isl_pos = isl_pos
min_size_allowed = int(img.pixel_beam()[0]*9.0)
if opts.rms_box is None or (opts.rms_box_bright is None and do_adapt):
if do_adapt:
bsize = int(max(brightsize, min_size_allowed, max_isl_size_highthresh*2.0))
else:
bsize = int(max(brightsize, min_size_allowed, max_isl_size*2.0))
bsize2 = int(max(min(image.shape)/10.0, max_isl_size*5.0))
if bsize < min_size_allowed:
bsize = min_size_allowed
if bsize % 10 == 0: bsize += 1
if bsize2 < min_size_allowed:
bsize2 = min_size_allowed
if bsize2 % 10 == 0: bsize2 += 1
bstep = int(round(min(bsize/3., min(shape)/10.)))
bstep2 = int(round(min(bsize2/3., min(shape)/10.)))
if opts.rms_box_bright is None:
img.rms_box_bright = (bsize, bstep)
else:
img.rms_box_bright = opts.rms_box_bright
if opts.rms_box is None:
img.rms_box = (bsize2, bstep2)
else:
img.rms_box = opts.rms_box
else:
if do_adapt:
img.rms_box_bright = opts.rms_box_bright
img.rms_box = opts.rms_box
else:
img.rms_box_bright = opts.rms_box
img.rms_box = opts.rms_box
if opts.kappa_clip is None:
kappa = -img.pixel_beamarea()
else:
kappa = img.opts.kappa_clip
if do_adapt:
map_opts = (kappa, img.rms_box_bright, opts.spline_rank)
else:
map_opts = (kappa, img.rms_box, opts.spline_rank)
for ipol, pol in enumerate(pols):
data = ch0_images[ipol]
mean = N.zeros(data.shape, dtype=N.float32)
rms = N.zeros(data.shape, dtype=N.float32)
if len(pols) > 1:
pol_txt = ' (' + pol + ')'
else:
pol_txt = ''
## calculate rms/mean maps if needed
if ((opts.rms_map is not False) or (opts.mean_map not in ['zero', 'const'])) and img.rms_box[0] > min(image.shape)/4.0:
# rms box is too large - just use constant rms and mean
self.output_rmsbox_size(img)
mylogger.userinfo(mylog, 'Size of rms_box larger than 1/4 of image size')
mylogger.userinfo(mylog, 'Using constant background rms and mean')
img.use_rms_map = False
img.mean_map_type = 'const'
else:
if (opts.rms_map is not False) or (opts.mean_map not in ['zero', 'const']):
if len(data.shape) == 2: ## 2d case
mean, rms = self.calculate_maps(img, data, mean, rms, mask, map_opts, do_adapt=do_adapt,
bright_pt_coords=isl_pos, rms_box2=img.rms_box,
logname="PyBDSM."+img.log, ncores=img.opts.ncores)
elif len(data.shape) == 3: ## 3d case
if not isinstance(mask, N.ndarray):
mask = N.zeros(data.shape[0], dtype=bool)
for i in range(data.shape[0]):
## iterate each plane
mean, rms = self.calculate_maps(img, data[i], mean[i], rms[i], mask[i], map_opts,
do_adapt=do_adapt, bright_pt_coords=isl_pos,
rms_box2=img.rms_box, logname="PyBDSM."+img.log,
ncores=img.opts.ncores)
else:
mylog.critical('Image shape not handleable' + pol_txt)
raise RuntimeError("Can't handle array of this shape" + pol_txt)
self.output_rmsbox_size(img)
if do_adapt:
mylogger.userinfo(mylog, 'Number of sources using small scale', str(len(isl_pos)))
mylog.info('Background rms and mean images computed' + pol_txt)
## check if variation of rms/mean maps is significant enough:
# check_rmsmap() sets img.use_rms_map
# check_meanmap() sets img.mean_map_type
if pol == 'I':
if opts.rms_map is None and img.use_rms_map is None:
if do_adapt and len(isl_pos) > 0:
# Always use 2d map if there is at least one bright
# source and adaptive scaling is desired
img.use_rms_map = True
else:
self.check_rmsmap(img, rms)
elif opts.rms_map != None:
img.use_rms_map = opts.rms_map
if img.use_rms_map is False:
mylogger.userinfo(mylog, 'Using constant background rms')
else:
mylogger.userinfo(mylog, 'Using 2D map for background rms')
if opts.mean_map == 'default' and img.mean_map_type is None:
self.check_meanmap(img, rms)
elif opts.mean_map != 'default':
img.mean_map_type = opts.mean_map
if img.mean_map_type != 'map':
mylogger.userinfo(mylog, 'Using constant background mean')
else:
mylogger.userinfo(mylog, 'Using 2D map for background mean')
## if rms map is insignificant, or rms_map==False use const value
if img.use_rms_map is False:
if opts.rms_value == None:
rms[:] = crmss[ipol]
else:
rms[:] = opts.rms_value
mylogger.userinfo(mylog, 'Value of background rms' + pol_txt,
'%.5f Jy/beam' % rms[0][0])
else:
rms_min = N.nanmin(rms)
rms_max = N.nanmax(rms)
mylogger.userinfo(mylog, 'Min/max values of background rms map' + pol_txt,
'(%.5f, %.5f) Jy/beam' % (rms_min, rms_max))
if img.mean_map_type != 'map':
if opts.mean_map == 'zero':
val = 0.0
else:
val = img.clipped_mean
mean[:] = val
mylogger.userinfo(mylog, 'Value of background mean' + pol_txt,
str(round(val,5))+' Jy/beam')
else:
mean_min = N.nanmin(mean)
mean_max = N.nanmax(mean)
mylogger.userinfo(mylog, 'Min/max values of background mean map' + pol_txt,
'(%.5f, %.5f) Jy/beam' % (mean_min, mean_max))
if pol == 'I':
# Apply mask to mean_map and rms_map by setting masked values to NaN
if isinstance(mask, N.ndarray):
pix_masked = N.where(mask == True)
mean[pix_masked] = N.nan
rms[pix_masked] = N.nan
img.mean_arr = mean
img.rms_arr = rms
if opts.savefits_rmsim or opts.output_all:
if img.waveletimage:
resdir = img.basedir + '/wavelet/background/'
else:
resdir = img.basedir + '/background/'
if not os.path.exists(resdir): os.makedirs(resdir)
func.write_image_to_file(img.use_io, img.imagename + '.rmsd_I.fits', rms, img, resdir)
mylog.info('%s %s' % ('Writing ', resdir+img.imagename+'.rmsd_I.fits'))
if opts.savefits_meanim or opts.output_all:
if img.waveletimage:
resdir = img.basedir + '/wavelet/background/'
else:
resdir = img.basedir + '/background/'
if not os.path.exists(resdir): os.makedirs(resdir)
func.write_image_to_file(img.use_io, img.imagename + '.mean_I.fits', mean, img, resdir)
mylog.info('%s %s' % ('Writing ', resdir+img.imagename+'.mean_I.fits'))
if opts.savefits_normim or opts.output_all:
if img.waveletimage:
resdir = img.basedir + '/wavelet/background/'
else:
resdir = img.basedir + '/background/'
if not os.path.exists(resdir): os.makedirs(resdir)
zero_pixels = N.where(rms <= 0.0)
rms_nonzero = rms.copy()
rms_nonzero[zero_pixels] = N.NaN
func.write_image_to_file(img.use_io, img.imagename + '.norm_I.fits', (image-mean)/rms_nonzero, img, resdir)
mylog.info('%s %s' % ('Writing ', resdir+img.imagename+'.norm_I.fits'))
else:
img.__setattr__('mean_'+pol+'_arr', mean)
img.__setattr__('rms_'+pol+'_arr', rms)
img.completed_Ops.append('rmsimage')
return img
def __call__(self, img):
import time, os
mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Readimage")
if img.opts.filename == '':
raise RuntimeError('Image file name not specified.')
# Check for trailing "/" in file name (since CASA images are directories).
# Although the general rule is to not alter the values in opts (only the
# user should be able to alter these), in this case there is no harm in
# replacing the file name in opts with the '/' trimmed off.
if img.opts.filename[-1] == '/':
img.opts.filename = img.opts.filename[:-1]
# Determine indir if not explicitly given by user (in img.opts.indir)
if img.opts.indir is None:
indir = os.path.dirname(img.opts.filename)
if indir == '':
indir = './'
img.indir = indir
else:
img.indir = img.opts.indir
# Try to trim common extensions from filename and store various
# paths
root, ext = os.path.splitext(img.opts.filename)
if ext in ['.fits', '.FITS', '.image']:
fname = root
elif ext in ['.gz', '.GZ']:
root2, ext2 = os.path.splitext(root)
if ext2 in ['.fits', '.FITS', '.image']:
fname = root2
else:
fname = root
else:
fname = img.opts.filename
img.filename = img.opts.filename
img.parentname = fname
img.imagename = fname + '.pybdsm'
img.basedir = './' + fname + '_pybdsm/'
# Read in data and header
image_file = os.path.basename(img.opts.filename)
result = read_image_from_file(image_file, img, img.indir)
if result is None:
raise RuntimeError("Cannot open file " + repr(image_file) + ". " + img._reason)
else:
data, hdr = result
# Check whether caching is to be used. If it is, set up a
# temporary directory. The temporary directory will be
# removed automatically upon exit.
if img.opts.do_cache:
img.do_cache = True
else:
img.do_cache = False
if img.do_cache:
mylog.info('Using disk caching.')
tmpdir = img.parentname+'_tmp'
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
img._tempdir_parent = TempDir(tmpdir)
img.tempdir = TempDir(tempfile.mkdtemp(dir=tmpdir))
import atexit, shutil
atexit.register(shutil.rmtree, img._tempdir_parent, ignore_errors=True)
else:
img.tempdir = None
# Store data and header in img. If polarisation_do = False, only store pol == 'I'
img.nchan = data.shape[1]
img.nstokes = data.shape[0]
mylogger.userinfo(mylog, 'Image size',
str(data.shape[-2:]) + ' pixels')
mylogger.userinfo(mylog, 'Number of channels',
'%i' % data.shape[1])
mylogger.userinfo(mylog, 'Number of Stokes parameters',
'%i' % data.shape[0])
if img.opts.polarisation_do and data.shape[0] == 1:
img.opts.polarisation_do = False
mylog.warning('Image has Stokes I only. Polarisation module disabled.')
if img.opts.polarisation_do or data.shape[0] == 1:
img.image_arr = data
else:
img.image_arr = data[0, :].reshape(1, data.shape[1], data.shape[2], data.shape[3])
img.header = hdr
img.shape = data.shape
img.j = 0
### initialize wcs conversion routines
self.init_wcs(img)
self.init_beam(img)
self.init_freq(img)
year, code = self.get_equinox(img)
if year is None:
mylog.info('Equinox not found in image header. Assuming J2000.')
img.equinox = 2000.0
else:
mylog.info('Equinox of image is %f.' % year)
img.equinox = year
if img.opts.output_all:
# Set up directory to write output to
opdir = img.opts.opdir_overwrite
if opdir not in ['overwrite', 'append']:
img.opts.opdir_overwrite = 'append'
if opdir == 'append':
mylog.info('Appending output files to directory ' + img.basedir)
else:
mylog.info('Overwriting output files (if any) in directory ' + img.basedir)
if os.path.isdir(img.basedir):
os.system("rm -fr " + img.basedir + '/*')
if not os.path.isdir(img.basedir):
os.makedirs(img.basedir)
# Now add solname (if any) and time to basedir
if img.opts.solnname is not None:
img.basedir += img.opts.solnname + '_'
img.basedir += time.strftime("%d%b%Y_%H.%M.%S")
# Make the final output directory
if not os.path.isdir(img.basedir):
os.makedirs(img.basedir)
del data
img.completed_Ops.append('readimage')
return img
def __call__(self, img):
if img.opts.psf_vary_do:
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Psf_Vary")
mylogger.userinfo(mylog, '\nEstimating PSF variations')
opts = img.opts
dir = img.basedir + '/misc/'
plot = False # debug figures
image = img.ch0_arr
try:
from astropy.io import fits as pyfits
old_pyfits = False
except ImportError, err:
from distutils.version import StrictVersion
import pyfits
if StrictVersion(pyfits.__version__) < StrictVersion('2.2'):
old_pyfits = True
else:
old_pyfits = False
if old_pyfits:
mylog.warning('PyFITS version is too old: psf_vary module skipped')
return
if opts.psf_fwhm is not None:
# User has specified a constant PSF to use, so skip PSF fitting/etc.
psf_maj = opts.psf_fwhm[0] # FWHM in deg
psf_min = opts.psf_fwhm[1] # FWHM in deg
psf_pa = opts.psf_fwhm[2] # PA in deg
mylogger.userinfo(mylog, 'Using constant PSF (major, minor, pos angle)',
'(%.5e, %.5e, %s) degrees' % (psf_maj, psf_maj,
round(psf_pa, 1)))
else:
# Use did not specify a constant PSF to use, so estimate it
over = 2
generators = opts.psf_generators; nsig = opts.psf_nsig; kappa2 = opts.psf_kappa2
snrtop = opts.psf_snrtop; snrbot = opts.psf_snrbot; snrcutstack = opts.psf_snrcutstack
gencode = opts.psf_gencode; primarygen = opts.psf_primarygen; itess_method = opts.psf_itess_method
tess_sc = opts.psf_tess_sc; tess_fuzzy= opts.psf_tess_fuzzy
bright_snr_cut = opts.psf_high_snr
s_only = opts.psf_stype_only
if opts.psf_snrcut < 5.0:
mylogger.userinfo(mylog, "Value of psf_snrcut too low; increasing to 5")
snrcut = 5.0
else:
snrcut = opts.psf_snrcut
img.psf_snrcut = snrcut
if opts.psf_high_snr is not None:
if opts.psf_high_snr < 10.0:
mylogger.userinfo(mylog, "Value of psf_high_snr too low; increasing to 10")
high_snrcut = 10.0
else:
high_snrcut = opts.psf_high_snr
else:
high_snrcut = opts.psf_high_snr
img.psf_high_snr = high_snrcut
wtfns=['unity', 'roundness', 'log10', 'sqrtlog10']
if 0 <= itess_method < 4: tess_method=wtfns[itess_method]
else: tess_method='unity'
### now put all relevant gaussian parameters into a list
ngaus = img.ngaus
nsrc = img.nsrc
num = N.zeros(nsrc, dtype=N.int32)
peak = N.zeros(nsrc)
xc = N.zeros(nsrc)
yc = N.zeros(nsrc)
bmaj = N.zeros(nsrc)
bmin = N.zeros(nsrc)
bpa = N.zeros(nsrc)
code = N.array(['']*nsrc);
rms = N.zeros(nsrc)
src_id_list = []
for i, src in enumerate(img.sources):
src_max = 0.0
for gmax in src.gaussians:
# Take only brightest Gaussian per source
if gmax.peak_flux > src_max:
src_max = gmax.peak_flux
g = gmax
num[i] = i
peak[i] = g.peak_flux
xc[i] = g.centre_pix[0]
yc[i] = g.centre_pix[1]
bmaj[i] = g.size_pix[0]
bmin[i] = g.size_pix[1]
bpa[i] = g.size_pix[2]
code[i] = img.sources[g.source_id].code
rms[i] = img.islands[g.island_id].rms
gauls = (num, peak, xc, yc, bmaj, bmin, bpa, code, rms)
tr_gauls = self.trans_gaul(gauls)
# takes gaussians with code=S and snr > snrcut.
if s_only:
tr = [n for n in tr_gauls if n[1]/n[8]>snrcut and n[7] == 'S']
else:
tr = [n for n in tr_gauls if n[1]/n[8]>snrcut]
g_gauls = self.trans_gaul(tr)
# computes statistics of fitted sizes. Same as psfvary_fullstat.f in fBDSM.
bmaj_a, bmaj_r, bmaj_ca, bmaj_cr, ni = _cbdsm.bstat(bmaj, None, nsig)
bmin_a, bmin_r, bmin_ca, bmin_cr, ni = _cbdsm.bstat(bmin, None, nsig)
bpa_a, bpa_r, bpa_ca, bpa_cr, ni = _cbdsm.bstat(bpa, None, nsig)
# get subset of sources deemed to be unresolved. Same as size_ksclip_wenss.f in fBDSM.
flag_unresolved = self.get_unresolved(g_gauls, img.beam, nsig, kappa2, over, img.psf_high_snr, plot)
if len(flag_unresolved) == 0:
mylog.warning('Insufficient number of sources to determine PSF variation.\nTry changing the PSF options or specify a (constant) PSF with the "psf_fwhm" option')
return
# see how much the SNR-weighted sizes of unresolved sources differ from the synthesized beam.
wtsize_beam_snr = self.av_psf(g_gauls, img.beam, flag_unresolved)
# filter out resolved sources
tr_gaul = self.trans_gaul(g_gauls)
tr = [n for i, n in enumerate(tr_gaul) if flag_unresolved[i]]
g_gauls = self.trans_gaul(tr)
mylogger.userinfo(mylog, 'Number of unresolved sources', str(len(g_gauls[0])))
# get a list of voronoi generators. vorogenS has values (and not None) if generators='field'.
vorogenP, vorogenS = self.get_voronoi_generators(g_gauls, generators, gencode, snrcut, snrtop, snrbot, snrcutstack)
mylogger.userinfo(mylog, 'Number of generators for PSF variation', str(len(vorogenP[0])))
if len(vorogenP[0]) < 3:
mylog.warning('Insufficient number of generators')
return
mylogger.userinfo(mylog, 'Tesselating image')
# group generators into tiles
tile_prop = self.edit_vorogenlist(vorogenP, frac=0.9)
# tesselate the image
volrank, vorowts = self.tesselate(vorogenP, vorogenS, tile_prop, tess_method, tess_sc, tess_fuzzy, \
generators, gencode, image.shape)
if opts.output_all:
func.write_image_to_file(img.use_io, img.imagename + '.volrank.fits', volrank, img, dir)
tile_list, tile_coord, tile_snr = tile_prop
ntile = len(tile_list)
bar = statusbar.StatusBar('Determining PSF variation ............... : ', 0, ntile)
mylogger.userinfo(mylog, 'Number of tiles for PSF variation', str(ntile))
# For each tile, calculate the weighted averaged psf image. Also for all the sources in the image.
cdelt = list(img.wcs_obj.acdelt[0:2])
factor=3.
psfimages, psfcoords, totpsfimage, psfratio, psfratio_aper = self.psf_in_tile(image, img.beam, g_gauls, \
cdelt, factor, snrcutstack, volrank, tile_prop, plot, img)
npsf = len(psfimages)
if opts.psf_use_shap:
if opts.psf_fwhm is None:
# use totpsfimage to get beta, centre and nmax for shapelet decomposition. Use nmax=5 or 6
mask=N.zeros(totpsfimage.shape, dtype=bool)
(m1, m2, m3)=func.moment(totpsfimage, mask)
betainit=sqrt(m3[0]*m3[1])*2.0 * 1.4
tshape = totpsfimage.shape
cen = N.array(N.unravel_index(N.argmax(totpsfimage), tshape))+[1,1]
cen = tuple(cen)
nmax = 12
basis = 'cartesian'
betarange = [0.5,sqrt(betainit*max(tshape))]
beta, error = sh.shape_varybeta(totpsfimage, mask, basis, betainit, cen, nmax, betarange, plot)
if error == 1: print ' Unable to find minimum in beta'
# decompose all the psf images using the beta from above
nmax=12; psf_cf=[]
for i in range(npsf):
psfim = psfimages[i]
cf = sh.decompose_shapelets(psfim, mask, basis, beta, cen, nmax, mode='')
psf_cf.append(cf)
if img.opts.quiet == False:
bar.increment()
bar.stop()
# transpose the psf image list
xt, yt = N.transpose(tile_coord)
tr_psf_cf = N.transpose(N.array(psf_cf))
# interpolate the coefficients across the image. Ok, interpolate in scipy for
# irregular grids is crap. doesnt even pass through some of the points.
# for now, fit polynomial.
compress = 100.0
x, y = N.transpose(psfcoords)
if len(x) < 3:
mylog.warning('Insufficient number of tiles to do interpolation of PSF variation')
return
psf_coeff_interp, xgrid, ygrid = self.interp_shapcoefs(nmax, tr_psf_cf, psfcoords, image.shape, \
compress, plot)
psfshape = psfimages[0].shape
skip = 5
aa = self.create_psf_grid(psf_coeff_interp, image.shape, xgrid, ygrid, skip, nmax, psfshape, \
basis, beta, cen, totpsfimage, plot)
img.psf_images = aa
else:
if opts.psf_fwhm is None:
if ntile < 4:
mylog.warning('Insufficient number of tiles to do interpolation of PSF variation')
return
else:
# Fit stacked PSFs with Gaussians and measure aperture fluxes
bm_pix = N.array([img.pixel_beam()[0]*fwsig, img.pixel_beam()[1]*fwsig, img.pixel_beam()[2]])
psf_maj = N.zeros(npsf)
psf_min = N.zeros(npsf)
psf_pa = N.zeros(npsf)
if img.opts.quiet == False:
bar.start()
for i in range(ntile):
psfim = psfimages[i]
mask = N.zeros(psfim.shape, dtype=bool)
x_ax, y_ax = N.indices(psfim.shape)
maxv = N.max(psfim)
p_ini = [maxv, (psfim.shape[0]-1)/2.0*1.1, (psfim.shape[1]-1)/2.0*1.1, bm_pix[0]/fwsig*1.3,
bm_pix[1]/fwsig*1.1, bm_pix[2]*2]
para, ierr = func.fit_gaus2d(psfim, p_ini, x_ax, y_ax, mask)
### first extent is major
if para[3] < para[4]:
para[3:5] = para[4:2:-1]
para[5] += 90
### clip position angle
para[5] = divmod(para[5], 180)[1]
psf_maj[i] = para[3]
psf_min[i] = para[4]
posang = para[5]
while posang >= 180.0:
posang -= 180.0
psf_pa[i] = posang
if img.opts.quiet == False:
bar.increment()
bar.stop()
# Interpolate Gaussian parameters
if img.aperture is None:
psf_maps = [psf_maj, psf_min, psf_pa, psfratio]
else:
psf_maps = [psf_maj, psf_min, psf_pa, psfratio, psfratio_aper]
nimgs = len(psf_maps)
bar = statusbar.StatusBar('Interpolating PSF images ................ : ', 0, nimgs)
if img.opts.quiet == False:
bar.start()
map_list = mp.parallel_map(func.eval_func_tuple,
itertools.izip(itertools.repeat(self.interp_prop),
psf_maps, itertools.repeat(psfcoords),
itertools.repeat(image.shape)), numcores=opts.ncores,
bar=bar)
if img.aperture is None:
psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int = map_list
else:
psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int, psf_ratio_aper_int = map_list
# Smooth if desired
if img.opts.psf_smooth is not None:
sm_scale = img.opts.psf_smooth / img.pix2beam([1.0, 1.0, 0.0])[0] / 3600.0 # pixels
if img.opts.aperture is None:
psf_maps = [psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int]
else:
psf_maps = [psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int, psf_ratio_aper_int]
nimgs = len(psf_maps)
bar = statusbar.StatusBar('Smoothing PSF images .................... : ', 0, nimgs)
if img.opts.quiet == False:
bar.start()
map_list = mp.parallel_map(func.eval_func_tuple,
itertools.izip(itertools.repeat(self.blur_image),
psf_maps, itertools.repeat(sm_scale)), numcores=opts.ncores,
bar=bar)
if img.aperture is None:
psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int = map_list
else:
psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int, psf_ratio_aper_int = map_list
# Make sure all smoothed, interpolated images are ndarrays
psf_maj_int = N.array(psf_maj_int)
psf_min_int = N.array(psf_min_int)
psf_pa_int = N.array(psf_pa_int)
psf_ratio_int = N.array(psf_ratio_int)
if img.aperture is None:
psf_ratio_aper_int = N.zeros(psf_maj_int.shape, dtype=N.float32)
else:
psf_ratio_aper_int = N.array(psf_ratio_aper_int, dtype=N.float32)
# Blank with NaNs if needed
mask = img.mask_arr
if isinstance(mask, N.ndarray):
pix_masked = N.where(mask == True)
psf_maj_int[pix_masked] = N.nan
psf_min_int[pix_masked] = N.nan
psf_pa_int[pix_masked] = N.nan
psf_ratio_int[pix_masked] = N.nan
psf_ratio_aper_int[pix_masked] = N.nan
# Store interpolated images. The major and minor axis images are
# the sigma in units of arcsec, the PA image in units of degrees east of
# north, the ratio images in units of 1/beam.
img.psf_vary_maj_arr = psf_maj_int * img.pix2beam([1.0, 1.0, 0.0])[0] * 3600.0 # sigma in arcsec
img.psf_vary_min_arr = psf_min_int * img.pix2beam([1.0, 1.0, 0.0])[0] * 3600.0 # sigma in arcsec
img.psf_vary_pa_arr = psf_pa_int
img.psf_vary_ratio_arr = psf_ratio_int # in 1/beam
img.psf_vary_ratio_aper_arr = psf_ratio_aper_int # in 1/beam
if opts.output_all:
func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_maj.fits', img.psf_vary_maj_arr*fwsig, img, dir)
func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_min.fits', img.psf_vary_min_arr*fwsig, img, dir)
func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_pa.fits', img.psf_vary_pa_arr, img, dir)
func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_ratio.fits', img.psf_vary_ratio_arr, img, dir)
func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_ratio_aper.fits', img.psf_vary_ratio_aper_arr, img, dir)
# Loop through source and Gaussian lists and deconvolve the sizes using appropriate beam
bar2 = statusbar.StatusBar('Correcting deconvolved source sizes ..... : ', 0, img.nsrc)
if img.opts.quiet == False:
bar2.start()
for src in img.sources:
src_pos = img.sky2pix(src.posn_sky_centroid)
src_pos_int = (int(src_pos[0]), int(src_pos[1]))
gaus_c = img.gaus2pix(src.size_sky, src.posn_sky_centroid)
if opts.psf_fwhm is None:
gaus_bm = [psf_maj_int[src_pos_int]*fwsig, psf_min_int[src_pos_int]*fwsig, psf_pa_int[src_pos_int]]
else:
# Use user-specified constant PSF instead
gaus_bm = img.beam2pix(opts.psf_fwhm)
gaus_dc, err = func.deconv2(gaus_bm, gaus_c)
src.deconv_size_sky = img.pix2gaus(gaus_dc, src_pos)
src.deconv_size_skyE = [0.0, 0.0, 0.0]
for g in src.gaussians:
gaus_c = img.gaus2pix(g.size_sky, src.posn_sky_centroid)
gaus_dc, err = func.deconv2(gaus_bm, gaus_c)
g.deconv_size_sky = img.pix2gaus(gaus_dc, g.centre_pix)
g.deconv_size_skyE = [0.0, 0.0, 0.0]
if img.opts.quiet == False:
bar2.spin()
if img.opts.quiet == False:
bar2.increment()
bar2.stop()
img.completed_Ops.append('psf_vary')
def __call__(self, img):
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Threshold ")
data = img.ch0_arr
mask = img.mask_arr
opts = img.opts
size = N.product(img.ch0_arr.shape)
sq2 = sqrt(2)
if img.opts.thresh is None:
source_p = self.get_srcp(img)
cutoff = 5.0
false_p = 0.5*erfc(cutoff/sq2)*size
if false_p < opts.fdr_ratio*source_p:
img.thresh = 'hard'
mylogger.userinfo(mylog, "Expected 5-sigma-clipped false detection rate < fdr_ratio")
mylogger.userinfo(mylog, "Using sigma-clipping ('hard') thresholding")
else:
img.thresh = 'fdr'
mylogger.userinfo(mylog, "Expected 5-sigma-clipped false detection rate > fdr_ratio")
mylogger.userinfo(mylog, "Using FDR (False Detection Rate) thresholding")
mylog.debug('%s %g' % ("Estimated number of source pixels (using sourcecounts.py) is ",source_p))
mylog.debug('%s %g' % ("Number of false positive pixels expected for 5-sigma is ",false_p))
mylog.debug("Threshold for pixels set to : "+str.swapcase(img.thresh))
else:
img.thresh = img.opts.thresh
if img.thresh=='fdr':
cdelt = img.wcs_obj.acdelt[:2]
bm = (img.beam[0], img.beam[1])
area_pix = int(round(N.product(bm)/(abs(N.product(cdelt))* \
pi/(4.0*log(2.0)))))
s0 = 0
for i in range(area_pix):
s0 += 1.0/(i+1)
slope = opts.fdr_alpha/s0
# sort erf of normalised image as vector
v = N.sort(0.5*erfc(N.ravel((data-img.mean_arr)/img.rms_arr)/sq2))[::-1]
pcrit = None
for i,x in enumerate(v):
if x < slope*i/size:
pcrit = x
break
if pcrit is None:
raise RuntimeError("FDR thresholding failed. Please check the input image for problems.")
dumr1 = 1.0-2.0*pcrit
dumr = 8.0/3.0/pi*(pi-3.0)/(4.0-pi)
# approx for inv(erfc)
sigcrit = sqrt(-2.0/pi/dumr-log(1.0-dumr1*dumr1)/2.0+ \
sqrt((2.0/pi/dumr+log(1.0-dumr1*dumr1)/2.0)* \
(2.0/pi/dumr+log(1.0-dumr1*dumr1)/2.0)- \
log(1.0-dumr1*dumr1)/dumr))*sq2
if pcrit == 0.0:
img.thresh = 'hard'
else:
img.thresh_pix = sigcrit
mylogger.userinfo(mylog, "FDR threshold (replaces thresh_pix)", str(round(sigcrit, 4)))
else:
img.thresh_pix = opts.thresh_pix
img.completed_Ops.append('threshold')
return img
def __call__(self, img):
mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Collapse")
if img.opts.polarisation_do:
pols = ['I', 'Q', 'U', 'V'] # make sure I is done first
else:
pols = ['I'] # assume I is always present
img.ch0_Q_arr = None
img.ch0_U_arr = None
img.ch0_V_arr = None
if img.shape[1] > 1:
c_mode = img.opts.collapse_mode
chan0 = img.opts.collapse_ch0
c_list = img.opts.collapse_av
c_wts = img.opts.collapse_wt
if c_list == []: c_list = N.arange(img.shape[1])
if len(c_list) == 1:
c_mode = 'single'
chan0 = c_list[0]
img.collapse_ch0 = chan0
ch0sh = img.image_arr.shape[2:]
if img.opts.polarisation_do:
ch0images = ['ch0_arr', 'ch0_Q_arr', 'ch0_U_arr', 'ch0_V_arr']
else:
ch0images = ['ch0_arr']
# assume all Stokes images have the same blank pixels as I:
blank = N.isnan(img.image_arr[0])
hasblanks = blank.any()
if img.opts.kappa_clip is None:
kappa = -img.pixel_beamarea()
else:
kappa = img.opts.kappa_clip
mean, rms, cmean, crms = chan_stats(img, kappa)
img.channel_mean = mean
img.channel_rms = rms
img.channel_clippedmean = cmean
img.channel_clippedrms = crms
for ipol, pol in enumerate(pols):
if c_mode == 'single':
if pol == 'I':
ch0 = img.image_arr[0, chan0]
img.ch0_arr = ch0
mylogger.userinfo(mylog, 'Source extraction will be ' \
'done on channel', '%i (%.3f MHz)' % \
(chan0, img.frequency/1e6))
else:
ch0[:] = img.image_arr[ipol, chan0][:]
img.__setattr__(ch0images[ipol][:], ch0)
if c_mode == 'average':
if not hasblanks:
if pol == 'I':
ch0, wtarr = avspc_direct(c_list, img.image_arr[0],
img.channel_clippedrms,
c_wts)
else:
# use wtarr from the I image, which is always collapsed first
ch0, wtarr = avspc_direct(c_list,
img.image_arr[ipol],
img.channel_clippedrms,
c_wts,
wtarr=wtarr)
else:
if pol == 'I':
ch0, wtarr = avspc_blanks(c_list, img.image_arr[0],
img.channel_clippedrms,
c_wts)
else:
# use wtarr from the I image, which is always collapsed first
ch0, wtarr = avspc_blanks(c_list,
img.image_arr[ipol],
img.channel_clippedrms,
c_wts,
wtarr=wtarr)
img.__setattr__(ch0images[ipol][:], ch0)
if pol == 'I':
img.avspc_wtarr = wtarr
init_freq_collapse(img, wtarr)
if c_wts == 'unity':
mylogger.userinfo(mylog, 'Channels averaged with '\
'uniform weights')
else:
mylogger.userinfo(mylog, 'Channels averaged with '\
'weights=(1/rms)^2')
mylogger.userinfo(mylog, 'Source extraction will be '\
'done on averaged ("ch0") image')
mylogger.userinfo(mylog, 'Frequency of averaged '\
'image', '%.3f MHz' % \
(img.frequency/1e6,))
str1 = " ".join(str(n) for n in c_list)
mylog.debug('%s %s' % ('Channels averaged : ', str1))
str1 = " ".join(["%9.4e" % n for n in wtarr])
mylog.debug('%s %s %s' %
('Channel weights : ', str1,
'; unity=zero if c_wts="rms"'))
if img.opts.output_all:
func.write_image_to_file(
img.use_io, img.imagename + '.ch0_' + pol + '.fits',
ch0, img)
mylog.debug('%s %s ' % ('Writing file ', img.imagename +
'.ch0_' + pol + '.fits'))
else:
# Only one channel in image
image = img.image_arr
img.ch0_arr = image[0, 0]
mylogger.userinfo(mylog, 'Frequency of image',
'%.3f MHz' % (img.frequency / 1e6, ))
if img.opts.polarisation_do:
for pol in pols[1:]:
if pol == 'Q':
img.ch0_Q_arr = image[1, 0][:]
if pol == 'U':
img.ch0_U_arr = image[2, 0][:]
if pol == 'V':
img.ch0_V_arr = image[3, 0][:]
# create mask if needed (assume all pols have the same mask as I)
image = img.ch0_arr
mask = N.isnan(image)
img.blankpix = N.sum(mask)
frac_blank = round(
float(img.blankpix) / float(image.shape[0] * image.shape[1]), 3)
mylogger.userinfo(
mylog, "Number of blank pixels",
str(img.blankpix) + ' (' + str(frac_blank * 100.0) + '%)')
# Check whether the input image might be an AWimage. If so, and there
# are no blank pixels, tell the user that they might to set blank_limit.
# Once the AWimager incorporates blanking, this check can be removed.
if img.opts.blank_limit is None and (
img.blankpix == 0 and
('restored' in img.filename.lower() or 'corr'
in img.filename.lower() or 'aw' in img.filename.lower())):
check_low = True
else:
check_low = False
if img.opts.blank_limit is not None or check_low:
import scipy
import sys
if check_low:
threshold = 1e-5
else:
threshold = img.opts.blank_limit
mylogger.userinfo(
mylog, "Blanking pixels with values "
"below %.1e Jy/beam" % (threshold, ))
bad = (abs(image) < threshold)
original_stdout = sys.stdout # keep a reference to STDOUT
sys.stdout = func.NullDevice() # redirect the real STDOUT
count = scipy.signal.convolve2d(bad, N.ones((3, 3)), mode='same')
sys.stdout = original_stdout # turn STDOUT back on
mask_low = (count >= 5)
if check_low:
nlow = len(N.where(mask_low)[0])
if nlow / float(image.shape[0] * image.shape[1]) > 0.2:
mylog.warn(
'A significant area of the image has very low values. To blank\nthese regions (e.g., because they are outside the primary beam), set the\nblank_limit option (values of 1e-5 to 1e-4 Jy/beam usually work well).\n'
)
else:
image[N.where(mask_low)] = N.nan
mask = N.isnan(image)
img.blankpix = N.sum(mask)
frac_blank = round(
float(img.blankpix) /
float(image.shape[0] * image.shape[1]), 3)
mylogger.userinfo(
mylog, "Total number of blanked pixels",
str(img.blankpix) + ' (' + str(frac_blank * 100.0) + '%)')
masked = mask.any()
img.masked = masked
if masked:
img.mask_arr = mask
else:
img.mask_arr = None
if img.blankpix == image.shape[0] * image.shape[1]:
# ALL pixels are blanked!
raise RuntimeError('All pixels in the image are blanked.')
img.completed_Ops.append('collapse')
def output_rmsbox_size(self, img):
"""Prints rms/mean box size"""
opts = img.opts
do_adapt = opts.adaptive_rms_box
mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "RMSimage")
if (opts.rms_map is not False) or (opts.mean_map
not in ['zero', 'const']):
if do_adapt:
if opts.rms_box_bright is None:
mylogger.userinfo(
mylog, 'Derived rms_box (box size, step size)',
'(' + str(img.rms_box_bright[0]) + ', ' +
str(img.rms_box_bright[1]) + ') pixels (small scale)')
else:
mylogger.userinfo(
mylog, 'Using user-specified rms_box',
'(' + str(img.rms_box_bright[0]) + ', ' +
str(img.rms_box_bright[1]) + ') pixels (small scale)')
if opts.rms_box is None:
mylogger.userinfo(
mylog, 'Derived rms_box (box size, step size)',
'(' + str(img.rms_box[0]) + ', ' +
str(img.rms_box[1]) + ') pixels (large scale)')
else:
mylogger.userinfo(
mylog, 'Using user-specified rms_box',
'(' + str(img.rms_box[0]) + ', ' +
str(img.rms_box[1]) + ') pixels (large scale)')
else:
if opts.rms_box is None:
mylogger.userinfo(
mylog, 'Derived rms_box (box size, step size)',
'(' + str(img.rms_box[0]) + ', ' +
str(img.rms_box[1]) + ') pixels')
else:
mylogger.userinfo(
mylog, 'Using user-specified rms_box',
'(' + str(img.rms_box[0]) + ', ' +
str(img.rms_box[1]) + ') pixels')
img, op_chain = get_op_chain(img)
if op_chain is not None:
_run_op_list(img, op_chain)
img._prev_opts = img.opts.to_dict()
return True
except RuntimeError, err:
# Catch and log error
mylog.error(str(err))
# Re-throw error if the user is not in the interactive shell
if img._is_interactive_shell:
return False
else:
raise
except KeyboardInterrupt:
mylogger.userinfo(mylog, "\n\033[31;1mAborted\033[0m")
return False
def get_op_chain(img):
"""Determines the optimal Op chain for an Image object.
This is useful when reprocessing an Image object. For example,
if Gaussians were already fit, but the user now wants to use
shapelets, we do not need to re-run Op_gausfit, etc.
Note that any new options added to opts.py should also be
added here. If not, a full reprocessing will be done if the
new option is changed.
"""
from . import default_chain
Op_chain = default_chain[:]
def __call__(self, img):
mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "RMSimage")
mylogger.userinfo(mylog, "Calculating background rms and mean images")
if img.opts.polarisation_do:
pols = ['I', 'Q', 'U', 'V']
ch0_images = [
img.ch0_arr, img.ch0_Q_arr, img.ch0_U_arr, img.ch0_V_arr
]
cmeans = [img.clipped_mean] + img.clipped_mean_QUV
crmss = [img.clipped_rms] + img.clipped_rms_QUV
else:
pols = ['I'] # assume I is always present
ch0_images = [img.ch0_arr]
cmeans = [img.clipped_mean]
crmss = [img.clipped_rms]
mask = img.mask_arr
opts = img.opts
cdelt = N.array(img.wcs_obj.acdelt[:2])
# Determine box size for rms/mean map calculations.
# If user specifies rms_box, use it. Otherwise, use either an
# adaptive binning scheme that shrinks the box near
# the brightest sources or estimate rms_box from bright sources.
#
# The adaptive scheme calculates the rms/mean map
# at two different scales:
# 1) using a large rms_box, set by size of largest source
# 2) using a small rms_box, set by size of largest bright source
# Then, the rms and mean values at a given point are determined
# by a weighted average of the values in the maps at the two
# scales.
fwsig = const.fwsig
min_adapt_threshold = 10.0
if opts.adaptive_thresh is None:
adapt_thresh = 50.0
start_thresh = 500.0
else:
adapt_thresh = opts.adaptive_thresh
if adapt_thresh < min_adapt_threshold:
adapt_thresh = min_adapt_threshold
opts.adaptive_thresh = min_adapt_threshold
start_thresh = adapt_thresh
brightsize = None
isl_pos = []
do_adapt = img.opts.adaptive_rms_box
img.use_rms_map = None
img.mean_map_type = None
# 'size' of brightest source
kappa1 = 3.0
try:
brightsize = int(
round(2. * img.beam[0] / cdelt[0] / fwsig *
sqrt(2. * log(img.max_value / (kappa1 * crms)))))
except:
brightsize = int(round(2. * img.beam[0] / cdelt[0] / fwsig))
mylog.info('Estimated size of brightest source (pixels) = ' +
str(brightsize))
# Using clipped mean and rms and a starting threshold of 500 sigma,
# search for bright sources. If fewer than 5 are found, reduce
# threshold until limit set by adapt_thresh is hit.
cmean = cmeans[0]
crms = crmss[0]
image = ch0_images[0]
shape = image.shape
isl_size_bright = []
isl_area_highthresh = []
isl_peak = []
max_isl_brightsize = 0.0
threshold = start_thresh
if do_adapt:
mylogger.userinfo(mylog, "Using adaptive scaling of rms_box")
while len(isl_size_bright) < 5 and threshold >= adapt_thresh:
isl_size_bright = []
isl_maxposn = []
act_pixels = (image - cmean) / threshold >= crms
threshold *= 0.8
if isinstance(mask, N.ndarray):
act_pixels[mask] = False
rank = len(image.shape)
connectivity = nd.generate_binary_structure(rank, rank)
labels, count = nd.label(act_pixels, connectivity)
slices = nd.find_objects(labels)
for idx, s in enumerate(slices):
isl_size_bright.append(
max([s[0].stop - s[0].start, s[1].stop - s[1].start]))
size_area = (labels[s]
== idx + 1).sum() / img.pixel_beamarea() * 2.0
isl_area_highthresh.append(size_area)
isl_maxposn.append(tuple(N.array(N.unravel_index(N.argmax(image[s]), image[s].shape))+\
N.array((s[0].start, s[1].start))))
isl_peak.append(nd.maximum(image[s], labels[s], idx + 1))
# Check islands found above at thresh_isl threshold to determine if
# the bright source is embedded inside a large island or not. If it is,
# exclude it from the bright-island list. Also find the size of the
# largest island at this threshold to set the large-scale rms_box
bright_threshold = threshold
threshold = 10.0
act_pixels = (image - cmean) / threshold >= crms
if isinstance(mask, N.ndarray):
act_pixels[mask] = False
rank = len(image.shape)
connectivity = nd.generate_binary_structure(rank, rank)
labels, count = nd.label(act_pixels, connectivity)
slices = nd.find_objects(labels)
isl_size = []
isl_size_highthresh = []
isl_size_lowthresh = []
isl_snr = []
thratio = threshold / bright_threshold
for idx, s in enumerate(slices):
isl_area_lowthresh = (labels[s] == idx +
1).sum() / img.pixel_beamarea() * 2.0
isl_maxposn_lowthresh = tuple(
N.array(N.unravel_index(N.argmax(image[s]), image[s].shape)) +
N.array((s[0].start, s[1].start)))
isl_size += [s[0].stop - s[0].start, s[1].stop - s[1].start]
if do_adapt and isl_maxposn_lowthresh in isl_maxposn:
bright_indx = isl_maxposn.index(isl_maxposn_lowthresh)
if isl_area_lowthresh < 25.0 or isl_area_lowthresh / isl_area_highthresh[
bright_indx] < 8.0:
isl_pos.append(isl_maxposn_lowthresh)
isl_size_lowthresh.append(
max([s[0].stop - s[0].start, s[1].stop - s[1].start]))
isl_size_highthresh.append(isl_size_bright[bright_indx])
isl_snr.append(isl_peak[bright_indx] / crms)
if len(isl_size) == 0:
max_isl_size = 0.0
else:
max_isl_size = max(isl_size)
mylog.info(
'Maximum extent of largest 10-sigma island using clipped rms (pixels) = '
+ str(max_isl_size))
if len(isl_size_highthresh) == 0:
max_isl_size_highthresh = 0.0
max_isl_size_lowthresh = 0.0
else:
max_isl_size_highthresh = max(isl_size_highthresh)
max_isl_size_lowthresh = max(isl_size_lowthresh)
avg_max_isl_size = (max_isl_size_highthresh +
max_isl_size_lowthresh) / 2.0
if hasattr(img, '_adapt_rms_isl_pos'):
isl_pos = img._adapt_rms_isl_pos # set isl_pos to existing value (for wavelet analysis)
if len(isl_pos) == 0:
# No bright sources found
do_adapt = False
else:
img._adapt_rms_isl_pos = isl_pos
min_size_allowed = int(img.pixel_beam()[0] * 9.0)
if opts.rms_box is None or (opts.rms_box_bright is None and do_adapt):
if do_adapt:
bsize = int(
max(brightsize, min_size_allowed,
max_isl_size_highthresh * 2.0))
else:
bsize = int(
max(brightsize, min_size_allowed, max_isl_size * 2.0))
bsize2 = int(max(min(image.shape) / 10.0, max_isl_size * 5.0))
if bsize < min_size_allowed:
bsize = min_size_allowed
if bsize % 10 == 0: bsize += 1
if bsize2 < min_size_allowed:
bsize2 = min_size_allowed
if bsize2 % 10 == 0: bsize2 += 1
bstep = int(round(min(bsize / 3., min(shape) / 10.)))
bstep2 = int(round(min(bsize2 / 3., min(shape) / 10.)))
if opts.rms_box_bright is None:
img.rms_box_bright = (bsize, bstep)
else:
img.rms_box_bright = opts.rms_box_bright
if opts.rms_box is None:
img.rms_box = (bsize2, bstep2)
else:
img.rms_box = opts.rms_box
else:
if do_adapt:
img.rms_box_bright = opts.rms_box_bright
img.rms_box = opts.rms_box
else:
img.rms_box_bright = opts.rms_box
img.rms_box = opts.rms_box
if opts.kappa_clip is None:
kappa = -img.pixel_beamarea()
else:
kappa = img.opts.kappa_clip
if do_adapt:
map_opts = (kappa, img.rms_box_bright, opts.spline_rank)
else:
map_opts = (kappa, img.rms_box, opts.spline_rank)
for ipol, pol in enumerate(pols):
data = ch0_images[ipol]
mean = N.zeros(data.shape, dtype=N.float32)
rms = N.zeros(data.shape, dtype=N.float32)
if len(pols) > 1:
pol_txt = ' (' + pol + ')'
else:
pol_txt = ''
## calculate rms/mean maps if needed
if ((opts.rms_map is not False) or
(opts.mean_map not in ['zero', 'const'])
) and img.rms_box[0] > min(image.shape) / 4.0:
# rms box is too large - just use constant rms and mean
self.output_rmsbox_size(img)
mylogger.userinfo(
mylog, 'Size of rms_box larger than 1/4 of image size')
mylogger.userinfo(mylog,
'Using constant background rms and mean')
img.use_rms_map = False
img.mean_map_type = 'const'
else:
if (opts.rms_map is not False) or (opts.mean_map
not in ['zero', 'const']):
if len(data.shape) == 2: ## 2d case
mean, rms = self.calculate_maps(
img,
data,
mean,
rms,
mask,
map_opts,
do_adapt=do_adapt,
bright_pt_coords=isl_pos,
rms_box2=img.rms_box,
logname="PyBDSM." + img.log,
ncores=img.opts.ncores)
elif len(data.shape) == 3: ## 3d case
if not isinstance(mask, N.ndarray):
mask = N.zeros(data.shape[0], dtype=bool)
for i in range(data.shape[0]):
## iterate each plane
mean, rms = self.calculate_maps(
img,
data[i],
mean[i],
rms[i],
mask[i],
map_opts,
do_adapt=do_adapt,
bright_pt_coords=isl_pos,
rms_box2=img.rms_box,
logname="PyBDSM." + img.log,
ncores=img.opts.ncores)
else:
mylog.critical('Image shape not handleable' + pol_txt)
raise RuntimeError("Can't handle array of this shape" +
pol_txt)
self.output_rmsbox_size(img)
if do_adapt:
mylogger.userinfo(
mylog, 'Number of sources using small scale',
str(len(isl_pos)))
mylog.info('Background rms and mean images computed' +
pol_txt)
## check if variation of rms/mean maps is significant enough:
# check_rmsmap() sets img.use_rms_map
# check_meanmap() sets img.mean_map_type
if pol == 'I':
if opts.rms_map is None and img.use_rms_map is None:
if do_adapt and len(isl_pos) > 0:
# Always use 2d map if there is at least one bright
# source and adaptive scaling is desired
img.use_rms_map = True
else:
self.check_rmsmap(img, rms)
elif opts.rms_map is not None:
img.use_rms_map = opts.rms_map
if img.use_rms_map is False:
mylogger.userinfo(mylog,
'Using constant background rms')
else:
mylogger.userinfo(mylog,
'Using 2D map for background rms')
if opts.mean_map == 'default' and img.mean_map_type is None:
self.check_meanmap(img, rms)
elif opts.mean_map != 'default':
img.mean_map_type = opts.mean_map
if img.mean_map_type != 'map':
mylogger.userinfo(mylog,
'Using constant background mean')
else:
mylogger.userinfo(mylog,
'Using 2D map for background mean')
## if rms map is insignificant, or rms_map==False use const value
if img.use_rms_map is False:
if opts.rms_value is None:
rms[:] = crmss[ipol]
else:
rms[:] = opts.rms_value
mylogger.userinfo(mylog, 'Value of background rms' + pol_txt,
'%.5f Jy/beam' % rms[0][0])
else:
rms_min = N.nanmin(rms)
rms_max = N.nanmax(rms)
mylogger.userinfo(
mylog, 'Min/max values of background rms map' + pol_txt,
'(%.5f, %.5f) Jy/beam' % (rms_min, rms_max))
if img.mean_map_type != 'map':
if opts.mean_map == 'zero':
val = 0.0
else:
val = img.clipped_mean
mean[:] = val
mylogger.userinfo(mylog, 'Value of background mean' + pol_txt,
str(round(val, 5)) + ' Jy/beam')
else:
mean_min = N.nanmin(mean)
mean_max = N.nanmax(mean)
mylogger.userinfo(
mylog, 'Min/max values of background mean map' + pol_txt,
'(%.5f, %.5f) Jy/beam' % (mean_min, mean_max))
if pol == 'I':
# Apply mask to mean_map and rms_map by setting masked values to NaN
if isinstance(mask, N.ndarray):
pix_masked = N.where(mask == True)
mean[pix_masked] = N.nan
rms[pix_masked] = N.nan
img.mean_arr = mean
img.rms_arr = rms
if opts.savefits_rmsim or opts.output_all:
if img.waveletimage:
resdir = img.basedir + '/wavelet/background/'
else:
resdir = img.basedir + '/background/'
if not os.path.exists(resdir): os.makedirs(resdir)
func.write_image_to_file(img.use_io,
img.imagename + '.rmsd_I.fits',
rms, img, resdir)
mylog.info(
'%s %s' %
('Writing ', resdir + img.imagename + '.rmsd_I.fits'))
if opts.savefits_meanim or opts.output_all:
if img.waveletimage:
resdir = img.basedir + '/wavelet/background/'
else:
resdir = img.basedir + '/background/'
if not os.path.exists(resdir): os.makedirs(resdir)
func.write_image_to_file(img.use_io,
img.imagename + '.mean_I.fits',
mean, img, resdir)
mylog.info(
'%s %s' %
('Writing ', resdir + img.imagename + '.mean_I.fits'))
if opts.savefits_normim or opts.output_all:
if img.waveletimage:
resdir = img.basedir + '/wavelet/background/'
else:
resdir = img.basedir + '/background/'
if not os.path.exists(resdir): os.makedirs(resdir)
zero_pixels = N.where(rms <= 0.0)
rms_nonzero = rms.copy()
rms_nonzero[zero_pixels] = N.NaN
func.write_image_to_file(img.use_io,
img.imagename + '.norm_I.fits',
(image - mean) / rms_nonzero, img,
resdir)
mylog.info(
'%s %s' %
('Writing ', resdir + img.imagename + '.norm_I.fits'))
else:
img.__setattr__('mean_' + pol + '_arr', mean)
img.__setattr__('rms_' + pol + '_arr', rms)
img.completed_Ops.append('rmsimage')
return img
def __call__(self, img):
mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Wavelet")
if img.opts.atrous_do:
if img.nisl == 0:
mylog.warning(
"No islands found. Skipping wavelet decomposition.")
img.completed_Ops.append('wavelet_atrous')
return
mylog.info(
"Decomposing gaussian residual image into a-trous wavelets")
bdir = img.basedir + '/wavelet/'
if img.opts.output_all:
if not os.path.isdir(bdir): os.makedirs(bdir)
if not os.path.isdir(bdir + '/residual/'):
os.makedirs(bdir + '/residual/')
if not os.path.isdir(bdir + '/model/'):
os.makedirs(bdir + '/model/')
dobdsm = img.opts.atrous_bdsm_do
filter = {
'tr': {
'size': 3,
'vec': [1. / 4, 1. / 2, 1. / 4],
'name': 'Triangle'
},
'b3': {
'size': 5,
'vec': [1. / 16, 1. / 4, 3. / 8, 1. / 4, 1. / 16],
'name': 'B3 spline'
}
}
if dobdsm: wchain, wopts = self.setpara_bdsm(img)
n, m = img.ch0_arr.shape
# Calculate residual image that results from normal (non-wavelet) Gaussian fitting
Op_make_residimage()(img)
resid = img.resid_gaus_arr
lpf = img.opts.atrous_lpf
if lpf not in ['b3', 'tr']: lpf = 'b3'
jmax = img.opts.atrous_jmax
l = len(filter[lpf]['vec']
) # 1st 3 is arbit and 2nd 3 is whats expected for a-trous
if jmax < 1 or jmax > 15: # determine jmax
# Check if largest island size is
# smaller than 1/3 of image size. If so, use it to determine jmax.
min_size = min(resid.shape)
max_isl_shape = (0, 0)
for isl in img.islands:
if isl.image.shape[0] * isl.image.shape[1] > max_isl_shape[
0] * max_isl_shape[1]:
max_isl_shape = isl.image.shape
if max_isl_shape != (
0, 0) and min(max_isl_shape) < min(resid.shape) / 3.0:
min_size = min(max_isl_shape) * 4.0
else:
min_size = min(resid.shape)
jmax = int(
floor(
log((min_size / 3.0 * 3.0 - l) /
(l - 1) + 1) / log(2.0) + 1.0)) + 1
if min_size * 0.55 <= (l + (l - 1) * (2**(jmax) - 1)):
jmax = jmax - 1
img.wavelet_lpf = lpf
img.wavelet_jmax = jmax
mylog.info("Using " + filter[lpf]['name'] +
' filter with J_max = ' + str(jmax))
img.atrous_islands = []
img.atrous_gaussians = []
img.atrous_sources = []
img.atrous_opts = []
img.resid_wavelets_arr = cp(img.resid_gaus_arr)
im_old = img.resid_wavelets_arr
total_flux = 0.0
ntot_wvgaus = 0
stop_wav = False
pix_masked = N.where(N.isnan(resid) == True)
jmin = 1
if img.opts.ncores is None:
numcores = 1
else:
numcores = img.opts.ncores
for j in range(jmin, jmax +
1): # extra +1 is so we can do bdsm on cJ as well
mylogger.userinfo(mylog, "\nWavelet scale #" + str(j))
im_new = self.atrous(im_old,
filter[lpf]['vec'],
lpf,
j,
numcores=numcores,
use_scipy_fft=img.opts.use_scipy_fft)
im_new[
pix_masked] = N.nan # since fftconvolve wont work with blanked pixels
if img.opts.atrous_sum:
w = im_new
else:
w = im_old - im_new
im_old = im_new
suffix = 'w' + ` j `
filename = img.imagename + '.atrous.' + suffix + '.fits'
if img.opts.output_all:
func.write_image_to_file('fits', filename, w, img, bdir)
mylog.info('%s %s' % ('Wrote ', img.imagename +
'.atrous.' + suffix + '.fits'))
# now do bdsm on each wavelet image.
if dobdsm:
wopts['filename'] = filename
wopts['basedir'] = bdir
box = img.rms_box[0]
y1 = (l + (l - 1) * (2**(j - 1) - 1))
bs = max(5 * y1, box) # changed from 10 to 5
if bs > min(n, m) / 2:
wopts['rms_map'] = False
wopts['mean_map'] = 'const'
wopts['rms_box'] = None
else:
wopts['rms_box'] = (bs, bs / 3)
if hasattr(img, '_adapt_rms_isl_pos'):
bs_bright = max(5 * y1, img.rms_box_bright[0])
if bs_bright < bs / 1.5:
wopts['adaptive_rms_box'] = True
wopts['rms_box_bright'] = (bs_bright,
bs_bright / 3)
else:
wopts['adaptive_rms_box'] = False
if j <= 3:
wopts['ini_gausfit'] = 'default'
else:
wopts['ini_gausfit'] = 'nobeam'
wid = (l + (l - 1) * (2**(j - 1) - 1)) # / 3.0
b1, b2 = img.pixel_beam()[0:2]
b1 = b1 * fwsig
b2 = b2 * fwsig
cdelt = img.wcs_obj.acdelt[:2]
wimg = Image(wopts)
wimg.beam = (sqrt(wid * wid + b1 * b1) * cdelt[0] * 2.0,
sqrt(wid * wid + b2 * b2) * cdelt[1] * 2.0,
0.0)
wimg.orig_beam = img.beam
wimg.pixel_beam = img.pixel_beam
wimg.pixel_beamarea = img.pixel_beamarea
wimg.log = 'Wavelet.'
wimg.basedir = img.basedir
wimg.extraparams['bbsprefix'] = suffix
wimg.extraparams['bbsname'] = img.imagename + '.wavelet'
wimg.extraparams['bbsappend'] = True
wimg.bbspatchnum = img.bbspatchnum
wimg.waveletimage = True
wimg.j = j
if hasattr(img, '_adapt_rms_isl_pos'):
wimg._adapt_rms_isl_pos = img._adapt_rms_isl_pos
self.init_image_simple(wimg, img, w, '.atrous.' + suffix)
for op in wchain:
op(wimg)
gc.collect()
if isinstance(op,
Op_islands) and img.opts.atrous_orig_isl:
if wimg.nisl > 0:
# Find islands that do not share any pixels with
# islands in original ch0 image.
good_isl = []
# Make original rank image boolean; rank counts from 0, with -1 being
# outside any island
orig_rankim_bool = N.array(img.pyrank + 1,
dtype=bool)
# Multiply rank images
old_islands = orig_rankim_bool * (wimg.pyrank +
1) - 1
# Exclude islands that don't overlap with a ch0 island.
valid_ids = set(old_islands.flatten())
for idx, wvisl in enumerate(wimg.islands):
if idx in valid_ids:
wvisl.valid = True
good_isl.append(wvisl)
else:
wvisl.valid = False
wimg.islands = good_isl
wimg.nisl = len(good_isl)
mylogger.userinfo(mylog,
"Number of islands found",
'%i' % wimg.nisl)
# Renumber islands:
for wvindx, wvisl in enumerate(wimg.islands):
wvisl.island_id = wvindx
if isinstance(op, Op_gausfit):
# If opts.atrous_orig_isl then exclude Gaussians outside of
# the original ch0 islands
nwvgaus = 0
if img.opts.atrous_orig_isl:
gaul = wimg.gaussians
tot_flux = 0.0
if img.ngaus == 0:
gaus_id = -1
else:
gaus_id = img.gaussians[-1].gaus_num
wvgaul = []
for g in gaul:
if not hasattr(g, 'valid'):
g.valid = False
if not g.valid:
try:
isl_id = img.pyrank[
int(g.centre_pix[0] + 1),
int(g.centre_pix[1] + 1)]
except IndexError:
isl_id = -1
if isl_id >= 0:
isl = img.islands[isl_id]
gcenter = (g.centre_pix[0] -
isl.origin[0],
g.centre_pix[1] -
isl.origin[1])
if not isl.mask_active[gcenter]:
gaus_id += 1
gcp = Gaussian(
img, g.parameters[:],
isl.island_id, gaus_id)
gcp.gaus_num = gaus_id
gcp.wisland_id = g.island_id
gcp.jlevel = j
g.valid = True
isl.gaul.append(gcp)
isl.ngaus += 1
img.gaussians.append(gcp)
nwvgaus += 1
tot_flux += gcp.total_flux
else:
g.valid = False
g.jlevel = 0
else:
g.valid = False
g.jlevel = 0
vg = []
for g in wimg.gaussians:
if g.valid:
vg.append(g)
wimg.gaussians = vg
mylogger.userinfo(
mylog, "Number of valid wavelet Gaussians",
str(nwvgaus))
else:
# Keep all Gaussians and merge islands that overlap
tot_flux = check_islands_for_overlap(img, wimg)
# Now renumber the islands and adjust the rank image before going to next wavelet image
renumber_islands(img)
total_flux += tot_flux
if img.opts.interactive and has_pl:
dc = '\033[34;1m'
nc = '\033[0m'
print dc + '--> Displaying islands and rms image...' + nc
if max(wimg.ch0_arr.shape) > 4096:
print dc + '--> Image is large. Showing islands only.' + nc
wimg.show_fit(rms_image=False,
mean_image=False,
ch0_image=False,
ch0_islands=True,
gresid_image=False,
sresid_image=False,
gmodel_image=False,
smodel_image=False,
pyramid_srcs=False)
else:
wimg.show_fit()
prompt = dc + "Press enter to continue or 'q' stop fitting wavelet images : " + nc
answ = raw_input_no_history(prompt)
while answ != '':
if answ == 'q':
img.wavelet_jmax = j
stop_wav = True
break
answ = raw_input_no_history(prompt)
if len(wimg.gaussians) > 0:
img.resid_wavelets_arr = self.subtract_wvgaus(
img.opts, img.resid_wavelets_arr, wimg.gaussians,
wimg.islands)
if img.opts.atrous_sum:
im_old = self.subtract_wvgaus(
img.opts, im_old, wimg.gaussians, wimg.islands)
if stop_wav == True:
break
pyrank = N.zeros(img.pyrank.shape, dtype=N.int32)
for i, isl in enumerate(img.islands):
isl.island_id = i
for g in isl.gaul:
g.island_id = i
for dg in isl.dgaul:
dg.island_id = i
pyrank[isl.bbox] += N.invert(isl.mask_active) * (i + 1)
pyrank -= 1 # align pyrank values with island ids and set regions outside of islands to -1
img.pyrank = pyrank
pdir = img.basedir + '/misc/'
img.ngaus += ntot_wvgaus
img.total_flux_gaus += total_flux
mylogger.userinfo(mylog,
"Total flux density in model on all scales",
'%.3f Jy' % img.total_flux_gaus)
if img.opts.output_all:
func.write_image_to_file('fits',
img.imagename + '.atrous.cJ.fits',
im_new, img, bdir)
mylog.info('%s %s' %
('Wrote ', img.imagename + '.atrous.cJ.fits'))
func.write_image_to_file(
'fits', img.imagename + '.resid_wavelets.fits',
(img.ch0_arr - img.resid_gaus_arr +
img.resid_wavelets_arr), img, bdir + '/residual/')
mylog.info('%s %s' %
('Wrote ', img.imagename + '.resid_wavelets.fits'))
func.write_image_to_file(
'fits', img.imagename + '.model_wavelets.fits',
(img.resid_gaus_arr - img.resid_wavelets_arr), img,
bdir + '/model/')
mylog.info('%s %s' %
('Wrote ', img.imagename + '.model_wavelets.fits'))
img.completed_Ops.append('wavelet_atrous')
def __call__(self, img):
if img.opts.psf_vary_do:
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Psf_Vary")
mylogger.userinfo(mylog, '\nEstimating PSF variations')
opts = img.opts
dir = img.basedir + '/misc/'
plot = False # debug figures
image = img.ch0_arr
try:
from astropy.io import fits as pyfits
old_pyfits = False
except ImportError, err:
from distutils.version import StrictVersion
import pyfits
if StrictVersion(pyfits.__version__) < StrictVersion('2.2'):
old_pyfits = True
else:
old_pyfits = False
if old_pyfits:
mylog.warning('PyFITS version is too old: psf_vary module skipped')
return
over = 2
generators = opts.psf_generators; nsig = opts.psf_nsig; kappa2 = opts.psf_kappa2
snrtop = opts.psf_snrtop; snrbot = opts.psf_snrbot; snrcutstack = opts.psf_snrcutstack
gencode = opts.psf_gencode; primarygen = opts.psf_primarygen; itess_method = opts.psf_itess_method
tess_sc = opts.psf_tess_sc; tess_fuzzy= opts.psf_tess_fuzzy
bright_snr_cut = opts.psf_high_snr
s_only = opts.psf_stype_only
if opts.psf_snrcut < 5.0:
mylogger.userinfo(mylog, "Value of psf_snrcut too low; increasing to 5")
snrcut = 5.0
else:
snrcut = opts.psf_snrcut
img.psf_snrcut = snrcut
if opts.psf_high_snr != None:
if opts.psf_high_snr < 10.0:
mylogger.userinfo(mylog, "Value of psf_high_snr too low; increasing to 10")
high_snrcut = 10.0
else:
high_snrcut = opts.psf_high_snr
else:
high_snrcut = opts.psf_high_snr
img.psf_high_snr = high_snrcut
wtfns=['unity', 'roundness', 'log10', 'sqrtlog10']
if 0 <= itess_method < 4: tess_method=wtfns[itess_method]
else: tess_method='unity'
### now put all relevant gaussian parameters into a list
ngaus = img.ngaus
nsrc = img.nsrc
num = N.zeros(nsrc, dtype=N.int32)
peak = N.zeros(nsrc)
xc = N.zeros(nsrc)
yc = N.zeros(nsrc)
bmaj = N.zeros(nsrc)
bmin = N.zeros(nsrc)
bpa = N.zeros(nsrc)
code = N.array(['']*nsrc);
rms = N.zeros(nsrc)
src_id_list = []
for i, src in enumerate(img.sources):
src_max = 0.0
for gmax in src.gaussians:
# Take only brightest Gaussian per source
if gmax.peak_flux > src_max:
src_max = gmax.peak_flux
g = gmax
num[i] = i
peak[i] = g.peak_flux
xc[i] = g.centre_pix[0]
yc[i] = g.centre_pix[1]
bmaj[i] = g.size_pix[0]
bmin[i] = g.size_pix[1]
bpa[i] = g.size_pix[2]
code[i] = img.sources[g.source_id].code
rms[i] = img.islands[g.island_id].rms
gauls = (num, peak, xc, yc, bmaj, bmin, bpa, code, rms)
tr_gauls = self.trans_gaul(gauls)
# takes gaussians with code=S and snr > snrcut.
if s_only:
tr = [n for n in tr_gauls if n[1]/n[8]>snrcut and n[7] == 'S']
else:
tr = [n for n in tr_gauls if n[1]/n[8]>snrcut]
g_gauls = self.trans_gaul(tr)
# computes statistics of fitted sizes. Same as psfvary_fullstat.f in fBDSM.
bmaj_a, bmaj_r, bmaj_ca, bmaj_cr, ni = _cbdsm.bstat(bmaj, None, nsig)
bmin_a, bmin_r, bmin_ca, bmin_cr, ni = _cbdsm.bstat(bmin, None, nsig)
bpa_a, bpa_r, bpa_ca, bpa_cr, ni = _cbdsm.bstat(bpa, None, nsig)
# get subset of sources deemed to be unresolved. Same as size_ksclip_wenss.f in fBDSM.
flag_unresolved = self.get_unresolved(g_gauls, img.beam, nsig, kappa2, over, img.psf_high_snr, plot)
# see how much the SNR-weighted sizes of unresolved sources differ from the synthesized beam.
wtsize_beam_snr = self.av_psf(g_gauls, img.beam, flag_unresolved)
# filter out resolved sources
tr_gaul = self.trans_gaul(g_gauls)
tr = [n for i, n in enumerate(tr_gaul) if flag_unresolved[i]]
g_gauls = self.trans_gaul(tr)
mylogger.userinfo(mylog, 'Number of unresolved sources', str(len(g_gauls[0])))
# get a list of voronoi generators. vorogenS has values (and not None) if generators='field'.
vorogenP, vorogenS = self.get_voronoi_generators(g_gauls, generators, gencode, snrcut, snrtop, snrbot, snrcutstack)
mylogger.userinfo(mylog, 'Number of generators for PSF variation', str(len(vorogenP[0])))
if len(vorogenP[0]) < 3:
mylog.warning('Insufficient number of generators')
return
mylogger.userinfo(mylog, 'Tesselating image')
# group generators into tiles
tile_prop = self.edit_vorogenlist(vorogenP, frac=0.9)
# tesselate the image
volrank, vorowts = self.tesselate(vorogenP, vorogenS, tile_prop, tess_method, tess_sc, tess_fuzzy, \
generators, gencode, image.shape)
if opts.output_all:
func.write_image_to_file(img.use_io, img.imagename + '.volrank.fits', volrank, img, dir)
tile_list, tile_coord, tile_snr = tile_prop
ntile = len(tile_list)
bar = statusbar.StatusBar('Determining PSF variation ............... : ', 0, ntile)
mylogger.userinfo(mylog, 'Number of tiles for PSF variation', str(ntile))
# For each tile, calculate the weighted averaged psf image. Also for all the sources in the image.
cdelt = list(img.wcs_obj.acdelt[0:2])
factor=3.
psfimages, psfcoords, totpsfimage, psfratio, psfratio_aper = self.psf_in_tile(image, img.beam, g_gauls, \
cdelt, factor, snrcutstack, volrank, tile_prop, plot, img)
npsf = len(psfimages)
if opts.psf_use_shap:
# use totpsfimage to get beta, centre and nmax for shapelet decomposition. Use nmax=5 or 6
mask=N.zeros(totpsfimage.shape, dtype=bool)
(m1, m2, m3)=func.moment(totpsfimage, mask)
betainit=sqrt(m3[0]*m3[1])*2.0 * 1.4
tshape = totpsfimage.shape
cen = N.array(N.unravel_index(N.argmax(totpsfimage), tshape))+[1,1]
cen = tuple(cen)
nmax = 12
basis = 'cartesian'
betarange = [0.5,sqrt(betainit*max(tshape))]
beta, error = sh.shape_varybeta(totpsfimage, mask, basis, betainit, cen, nmax, betarange, plot)
if error == 1: print ' Unable to find minimum in beta'
# decompose all the psf images using the beta from above
nmax=12; psf_cf=[]
for i in range(npsf):
psfim = psfimages[i]
cf = sh.decompose_shapelets(psfim, mask, basis, beta, cen, nmax, mode='')
psf_cf.append(cf)
if img.opts.quiet == False:
bar.increment()
bar.stop()
# transpose the psf image list
xt, yt = N.transpose(tile_coord)
tr_psf_cf = N.transpose(N.array(psf_cf))
# interpolate the coefficients across the image. Ok, interpolate in scipy for
# irregular grids is crap. doesnt even pass through some of the points.
# for now, fit polynomial.
compress = 100.0
x, y = N.transpose(psfcoords)
if len(x) < 3:
mylog.warning('Insufficient number of tiles to do interpolation of PSF variation')
return
psf_coeff_interp, xgrid, ygrid = self.interp_shapcoefs(nmax, tr_psf_cf, psfcoords, image.shape, \
compress, plot)
psfshape = psfimages[0].shape
skip = 5
aa = self.create_psf_grid(psf_coeff_interp, image.shape, xgrid, ygrid, skip, nmax, psfshape, \
basis, beta, cen, totpsfimage, plot)
img.psf_images = aa
else:
if ntile < 4:
mylog.warning('Insufficient number of tiles to do interpolation of PSF variation')
return
else:
# Fit stacked PSFs with Gaussians and measure aperture fluxes
bm_pix = N.array([img.pixel_beam()[0]*fwsig, img.pixel_beam()[1]*fwsig, img.pixel_beam()[2]])
psf_maj = N.zeros(npsf)
psf_min = N.zeros(npsf)
psf_pa = N.zeros(npsf)
if img.opts.quiet == False:
bar.start()
for i in range(ntile):
psfim = psfimages[i]
mask = N.zeros(psfim.shape, dtype=bool)
x_ax, y_ax = N.indices(psfim.shape)
maxv = N.max(psfim)
p_ini = [maxv, (psfim.shape[0]-1)/2.0*1.1, (psfim.shape[1]-1)/2.0*1.1, bm_pix[0]/fwsig*1.3,
bm_pix[1]/fwsig*1.1, bm_pix[2]*2]
para, ierr = func.fit_gaus2d(psfim, p_ini, x_ax, y_ax, mask)
### first extent is major
if para[3] < para[4]:
para[3:5] = para[4:2:-1]
para[5] += 90
### clip position angle
para[5] = divmod(para[5], 180)[1]
psf_maj[i] = para[3]
psf_min[i] = para[4]
posang = para[5]
while posang >= 180.0:
posang -= 180.0
psf_pa[i] = posang
if img.opts.quiet == False:
bar.increment()
bar.stop()
# Interpolate Gaussian parameters
if img.aperture == None:
psf_maps = [psf_maj, psf_min, psf_pa, psfratio]
else:
psf_maps = [psf_maj, psf_min, psf_pa, psfratio, psfratio_aper]
nimgs = len(psf_maps)
bar = statusbar.StatusBar('Interpolating PSF images ................ : ', 0, nimgs)
if img.opts.quiet == False:
bar.start()
map_list = mp.parallel_map(func.eval_func_tuple,
itertools.izip(itertools.repeat(self.interp_prop),
psf_maps, itertools.repeat(psfcoords),
itertools.repeat(image.shape)), numcores=opts.ncores,
bar=bar)
if img.aperture == None:
psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int = map_list
else:
psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int, psf_ratio_aper_int = map_list
# Smooth if desired
if img.opts.psf_smooth != None:
sm_scale = img.opts.psf_smooth / img.pix2beam([1.0, 1.0, 0.0])[0] / 3600.0 # pixels
if img.opts.aperture == None:
psf_maps = [psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int]
else:
psf_maps = [psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int, psf_ratio_aper_int]
nimgs = len(psf_maps)
bar = statusbar.StatusBar('Smoothing PSF images .................... : ', 0, nimgs)
if img.opts.quiet == False:
bar.start()
map_list = mp.parallel_map(func.eval_func_tuple,
itertools.izip(itertools.repeat(self.blur_image),
psf_maps, itertools.repeat(sm_scale)), numcores=opts.ncores,
bar=bar)
if img.aperture == None:
psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int = map_list
else:
psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int, psf_ratio_aper_int = map_list
# Make sure all smoothed, interpolated images are ndarrays
psf_maj_int = N.array(psf_maj_int)
psf_min_int = N.array(psf_min_int)
psf_pa_int = N.array(psf_pa_int)
psf_ratio_int = N.array(psf_ratio_int)
if img.aperture == None:
psf_ratio_aper_int = N.zeros(psf_maj_int.shape, dtype=N.float32)
else:
psf_ratio_aper_int = N.array(psf_ratio_aper_int, dtype=N.float32)
# Blank with NaNs if needed
mask = img.mask_arr
if isinstance(mask, N.ndarray):
pix_masked = N.where(mask == True)
psf_maj_int[pix_masked] = N.nan
psf_min_int[pix_masked] = N.nan
psf_pa_int[pix_masked] = N.nan
psf_ratio_int[pix_masked] = N.nan
psf_ratio_aper_int[pix_masked] = N.nan
# Store interpolated images. The major and minor axis images are
# the sigma in units of arcsec, the PA image in units of degrees east of
# north, the ratio images in units of 1/beam.
img.psf_vary_maj_arr = psf_maj_int * img.pix2beam([1.0, 1.0, 0.0])[0] * 3600.0 # sigma in arcsec
img.psf_vary_min_arr = psf_min_int * img.pix2beam([1.0, 1.0, 0.0])[0] * 3600.0 # sigma in arcsec
img.psf_vary_pa_arr = psf_pa_int
img.psf_vary_ratio_arr = psf_ratio_int # in 1/beam
img.psf_vary_ratio_aper_arr = psf_ratio_aper_int # in 1/beam
if opts.output_all:
func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_maj.fits', img.psf_vary_maj_arr*fwsig, img, dir)
func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_min.fits', img.psf_vary_min_arr*fwsig, img, dir)
func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_pa.fits', img.psf_vary_pa_arr, img, dir)
func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_ratio.fits', img.psf_vary_ratio_arr, img, dir)
func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_ratio_aper.fits', img.psf_vary_ratio_aper_arr, img, dir)
# Loop through source and Gaussian lists and deconvolve the sizes using appropriate beam
bar2 = statusbar.StatusBar('Correcting deconvolved source sizes ..... : ', 0, img.nsrc)
if img.opts.quiet == False:
bar2.start()
for src in img.sources:
src_pos = img.sky2pix(src.posn_sky_centroid)
src_pos_int = (int(src_pos[0]), int(src_pos[1]))
gaus_c = img.gaus2pix(src.size_sky, src.posn_sky_centroid)
gaus_bm = [psf_maj_int[src_pos_int]*fwsig, psf_min_int[src_pos_int]*fwsig, psf_pa_int[src_pos_int]*fwsig]
gaus_dc, err = func.deconv2(gaus_bm, gaus_c)
src.deconv_size_sky = img.pix2gaus(gaus_dc, src_pos)
src.deconv_size_skyE = [0.0, 0.0, 0.0]
for g in src.gaussians:
gaus_c = img.gaus2pix(g.size_sky, src.posn_sky_centroid)
gaus_dc, err = func.deconv2(gaus_bm, gaus_c)
g.deconv_size_sky = img.pix2gaus(gaus_dc, g.centre_pix)
g.deconv_size_skyE = [0.0, 0.0, 0.0]
if img.opts.quiet == False:
bar2.spin()
if img.opts.quiet == False:
bar2.increment()
bar2.stop()
img.completed_Ops.append('psf_vary')
def __call__(self, img):
mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Polarisatn")
if img.opts.polarisation_do:
mylog.info('Extracting polarisation properties for all sources')
pols = ['I', 'Q', 'U', 'V']
# Run gausfit and gual2srl on PI image to look for polarized sources
# undetected in I
fit_PI = img.opts.pi_fit
n_new = 0
ch0_pi = N.sqrt(img.ch0_Q_arr**2 + img.ch0_U_arr**2)
img.ch0_pi_arr = ch0_pi
if fit_PI:
from . import _run_op_list
mylogger.userinfo(mylog, "\nChecking PI image for new sources")
mask = img.mask_arr
minsize = img.opts.minpix_isl
# Set up image object for PI image.
pi_chain, pi_opts = self.setpara_bdsm(img)
pimg = Image(pi_opts)
pimg.beam = img.beam
pimg.pixel_beam = img.pixel_beam
pimg.pixel_beamarea = img.pixel_beamarea
pimg.log = 'PI.'
pimg.pix2beam = img.pix2beam
pimg.beam2pix = img.beam2pix
pimg.pix2gaus = img.pix2gaus
pimg.gaus2pix = img.gaus2pix
pimg.pix2sky = img.pix2sky
pimg.sky2pix = img.sky2pix
pimg.pix2coord = img.pix2coord
pimg.wcs_obj = img.wcs_obj
pimg.mask_arr = mask
pimg.masked = img.masked
pimg.ch0_arr = ch0_pi
pimg._pi = True
success = _run_op_list(pimg, pi_chain)
if not success:
return
img.pi_islands = pimg.islands
img.pi_gaussians = pimg.gaussians
img.pi_sources = pimg.sources
# Now check for new sources in the PI image that are not
# found in the Stokes I image. If any new sources are found,
# adjust their IDs to follow after those found in I.
new_isl = []
new_src = []
new_gaus = []
n_new_src = 0
isl_id = img.islands[-1].island_id
src_id = img.sources[-1].source_id
gaus_id = img.gaussians[-1].gaus_num
for pi_isl in pimg.islands:
new_sources = []
for pi_src in pi_isl.sources:
if img.pyrank[int(img.sky2pix(pi_src.posn_sky_max)[0]),
int(img.sky2pix(pi_src.posn_sky_max)[1]
)] == -1:
src_id += 1
pi_src._pi = True
pi_src.island_id = isl_id
pi_src.source_id = src_id
pi_src.spec_indx = N.NaN
pi_src.e_spec_indx = N.NaN
pi_src.spec_norm = N.NaN
pi_src.specin_flux = [N.NaN]
pi_src.specin_fluxE = [N.NaN]
pi_src.specin_freq = [N.NaN]
pi_src.specin_freq0 = N.NaN
new_sources.append(pi_src)
new_src.append(pi_src)
n_new_src += 1
for g in pi_src.gaussians:
gaus_id += 1
new_gaus.append(g)
g.gaus_num = gaus_id
if len(new_sources) > 0:
isl_id += 1
pi_isl.sources = new_sources
pi_isl.island_id = isl_id
pi_isl._pi = True
new_isl.append(pi_isl)
n_new = len(new_isl)
mylogger.userinfo(mylog, "New sources found in PI image",
'%i (%i total)' % (n_new, img.nsrc + n_new))
if n_new > 0:
img.islands += new_isl
img.sources += new_src
img.gaussians += new_gaus
img.nsrc += n_new_src
bar = statusbar.StatusBar(
'Calculating polarisation properties .... : ', 0, img.nsrc)
if img.opts.quiet == False:
bar.start()
for isl in img.islands:
isl_bbox = isl.bbox
ch0_I = img.ch0_arr[isl_bbox]
ch0_Q = img.ch0_Q_arr[isl_bbox]
ch0_U = img.ch0_U_arr[isl_bbox]
ch0_V = img.ch0_V_arr[isl_bbox]
ch0_images = [ch0_I, ch0_Q, ch0_U, ch0_V]
for i, src in enumerate(isl.sources):
# For each source, assume the morphology does not change
# across the Stokes cube. This assumption allows us to fit
# the Gaussians of each source to each Stokes image by
# simply fitting only the overall normalizations of the
# individual Gaussians.
#
# First, fit all source Gaussians to each Stokes image:
x, y = N.mgrid[isl_bbox]
gg = src.gaussians
fitfix = N.ones(len(gg)) # fit only normalization
srcmask = isl.mask_active
total_flux = N.zeros(
(4, len(fitfix)), dtype=N.float32
) # array of fluxes: N_Stokes x N_Gaussians
errors = N.zeros(
(4, len(fitfix)), dtype=N.float32
) # array of fluxes: N_Stokes x N_Gaussians
for sind, image in enumerate(ch0_images):
if (sind == 0 and hasattr(src, '_pi')
) or sind > 0: # Fit I only for PI sources
p, ep = func.fit_mulgaus2d(image, gg, x, y,
srcmask, fitfix)
for ig in range(len(fitfix)):
center_pix = (p[ig * 6 + 1], p[ig * 6 + 2])
bm_pix = N.array([
img.pixel_beam()[0],
img.pixel_beam()[1],
img.pixel_beam()[2]
])
total_flux[sind, ig] = p[ig * 6] * p[
ig * 6 + 3] * p[ig * 6 + 4] / (bm_pix[0] *
bm_pix[1])
p = N.insert(p,
N.arange(len(fitfix)) * 6 + 6,
total_flux[sind])
if sind > 0:
rms_img = img.__getattribute__('rms_' +
pols[sind] +
'_arr')
else:
rms_img = img.rms_arr
if len(rms_img.shape) > 1:
rms_isl = rms_img[isl.bbox].mean()
else:
rms_isl = rms_img
errors[sind] = func.get_errors(img, p, rms_isl)[6]
# Now, assign fluxes to each Gaussian.
src_flux_I = 0.0
src_flux_Q = 0.0
src_flux_U = 0.0
src_flux_V = 0.0
src_flux_I_err_sq = 0.0
src_flux_Q_err_sq = 0.0
src_flux_U_err_sq = 0.0
src_flux_V_err_sq = 0.0
for ig, gaussian in enumerate(src.gaussians):
flux_I = total_flux[0, ig]
flux_I_err = abs(errors[0, ig])
flux_Q = total_flux[1, ig]
flux_Q_err = abs(errors[1, ig])
flux_U = total_flux[2, ig]
flux_U_err = abs(errors[2, ig])
flux_V = total_flux[3, ig]
flux_V_err = abs(errors[3, ig])
if hasattr(src, '_pi'):
gaussian.total_flux = flux_I
gaussian.total_fluxE = flux_I_err
gaussian.total_flux_Q = flux_Q
gaussian.total_flux_U = flux_U
gaussian.total_flux_V = flux_V
gaussian.total_fluxE_Q = flux_Q_err
gaussian.total_fluxE_U = flux_U_err
gaussian.total_fluxE_V = flux_V_err
if hasattr(src, '_pi'):
src_flux_I += flux_I
src_flux_I_err_sq += flux_I_err**2
src_flux_Q += flux_Q
src_flux_U += flux_U
src_flux_V += flux_V
src_flux_Q_err_sq += flux_Q_err**2
src_flux_U_err_sq += flux_U_err**2
src_flux_V_err_sq += flux_V_err**2
# Calculate and store polarisation fractions and angle for each Gaussian in the island
# For this we need the I flux, which we can just take from g.total_flux and src.total_flux
flux_I = gaussian.total_flux
flux_I_err = gaussian.total_fluxE
stokes = [flux_I, flux_Q, flux_U, flux_V]
stokes_err = [
flux_I_err, flux_Q_err, flux_U_err, flux_V_err
]
lpol_frac, lpol_frac_loerr, lpol_frac_hierr = self.calc_lpol_fraction(
stokes, stokes_err) # linear pol fraction
lpol_ang, lpol_ang_err = self.calc_lpol_angle(
stokes, stokes_err) # linear pol angle
cpol_frac, cpol_frac_loerr, cpol_frac_hierr = self.calc_cpol_fraction(
stokes, stokes_err) # circular pol fraction
tpol_frac, tpol_frac_loerr, tpol_frac_hierr = self.calc_tpol_fraction(
stokes, stokes_err) # total pol fraction
gaussian.lpol_fraction = lpol_frac
gaussian.lpol_fraction_loerr = lpol_frac_loerr
gaussian.lpol_fraction_hierr = lpol_frac_hierr
gaussian.cpol_fraction = cpol_frac
gaussian.cpol_fraction_loerr = cpol_frac_loerr
gaussian.cpol_fraction_hierr = cpol_frac_hierr
gaussian.tpol_fraction = tpol_frac
gaussian.tpol_fraction_loerr = tpol_frac_loerr
gaussian.tpol_fraction_hierr = tpol_frac_hierr
gaussian.lpol_angle = lpol_ang
gaussian.lpol_angle_err = lpol_ang_err
# Store fluxes for each source in the island
if hasattr(src, '_pi'):
src.total_flux = src_flux_I
src.total_fluxE = N.sqrt(src_flux_I_err_sq)
src.total_flux_Q = src_flux_Q
src.total_flux_U = src_flux_U
src.total_flux_V = src_flux_V
src.total_fluxE_Q = N.sqrt(src_flux_Q_err_sq)
src.total_fluxE_U = N.sqrt(src_flux_U_err_sq)
src.total_fluxE_V = N.sqrt(src_flux_V_err_sq)
# Calculate and store polarisation fractions and angle for each source in the island
# For this we need the I flux, which we can just take from g.total_flux and src.total_flux
src_flux_I = src.total_flux
src_flux_I_err = src.total_fluxE
stokes = [src_flux_I, src_flux_Q, src_flux_U, src_flux_V]
stokes_err = [
src_flux_I_err,
N.sqrt(src_flux_Q_err_sq),
N.sqrt(src_flux_U_err_sq),
N.sqrt(src_flux_V_err_sq)
]
lpol_frac, lpol_frac_loerr, lpol_frac_hierr = self.calc_lpol_fraction(
stokes, stokes_err) # linear pol fraction
lpol_ang, lpol_ang_err = self.calc_lpol_angle(
stokes, stokes_err) # linear pol angle
cpol_frac, cpol_frac_loerr, cpol_frac_hierr = self.calc_cpol_fraction(
stokes, stokes_err) # circular pol fraction
tpol_frac, tpol_frac_loerr, tpol_frac_hierr = self.calc_tpol_fraction(
stokes, stokes_err) # total pol fraction
src.lpol_fraction = lpol_frac
src.lpol_fraction_loerr = lpol_frac_loerr
src.lpol_fraction_hierr = lpol_frac_hierr
src.cpol_fraction = cpol_frac
src.cpol_fraction_loerr = cpol_frac_loerr
src.cpol_fraction_hierr = cpol_frac_hierr
src.tpol_fraction = tpol_frac
src.tpol_fraction_loerr = tpol_frac_loerr
src.tpol_fraction_hierr = tpol_frac_hierr
src.lpol_angle = lpol_ang
src.lpol_angle_err = lpol_ang_err
if bar.started:
bar.increment()
bar.stop()
img.completed_Ops.append('polarisation')
def __call__(self, img):
import time, os
mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Readimage")
if img.opts.filename == '':
raise RuntimeError('Image file name not specified.')
# Check for trailing "/" in file name (since CASA images are directories).
# Although the general rule is to not alter the values in opts (only the
# user should be able to alter these), in this case there is no harm in
# replacing the file name in opts with the '/' trimmed off.
if img.opts.filename[-1] == '/':
img.opts.filename = img.opts.filename[:-1]
# Determine indir if not explicitly given by user (in img.opts.indir)
if img.opts.indir is None:
indir = os.path.dirname(img.opts.filename)
if indir == '':
indir = './'
img.indir = indir
else:
img.indir = img.opts.indir
# Try to trim common extensions from filename and store various
# paths
root, ext = os.path.splitext(img.opts.filename)
if ext in ['.fits', '.FITS', '.image']:
fname = root
elif ext in ['.gz', '.GZ']:
root2, ext2 = os.path.splitext(root)
if ext2 in ['.fits', '.FITS', '.image']:
fname = root2
else:
fname = root
else:
fname = img.opts.filename
img.filename = img.opts.filename
img.parentname = fname
img.imagename = fname + '.pybdsm'
img.basedir = './' + fname + '_pybdsm/'
# Read in data and header
image_file = os.path.basename(img.opts.filename)
result = read_image_from_file(image_file, img, img.indir)
if result is None:
raise RuntimeError("Cannot open file " + repr(image_file) + ". " +
img._reason)
else:
data, hdr = result
# Check whether caching is to be used. If it is, set up a
# temporary directory. The temporary directory will be
# removed automatically upon exit.
if img.opts.do_cache:
img.do_cache = True
else:
img.do_cache = False
if img.do_cache:
mylog.info('Using disk caching.')
tmpdir = img.parentname + '_tmp'
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
img._tempdir_parent = TempDir(tmpdir)
img.tempdir = TempDir(tempfile.mkdtemp(dir=tmpdir))
import atexit, shutil
atexit.register(shutil.rmtree,
img._tempdir_parent,
ignore_errors=True)
else:
img.tempdir = None
# Store data and header in img. If polarisation_do = False, only store pol == 'I'
img.nchan = data.shape[1]
img.nstokes = data.shape[0]
mylogger.userinfo(mylog, 'Image size',
str(data.shape[-2:]) + ' pixels')
mylogger.userinfo(mylog, 'Number of channels', '%i' % data.shape[1])
mylogger.userinfo(mylog, 'Number of Stokes parameters',
'%i' % data.shape[0])
if img.opts.polarisation_do and data.shape[0] == 1:
img.opts.polarisation_do = False
mylog.warning(
'Image has Stokes I only. Polarisation module disabled.')
if img.opts.polarisation_do or data.shape[0] == 1:
img.image_arr = data
else:
img.image_arr = data[0, :].reshape(1, data.shape[1], data.shape[2],
data.shape[3])
img.header = hdr
img.shape = data.shape
img.j = 0
### initialize wcs conversion routines
self.init_wcs(img)
self.init_beam(img)
self.init_freq(img)
year, code = self.get_equinox(img)
if year is None:
mylog.info('Equinox not found in image header. Assuming J2000.')
img.equinox = 2000.0
else:
mylog.info('Equinox of image is %f.' % year)
img.equinox = year
if img.opts.output_all:
# Set up directory to write output to
opdir = img.opts.opdir_overwrite
if opdir not in ['overwrite', 'append']:
img.opts.opdir_overwrite = 'append'
if opdir == 'append':
mylog.info('Appending output files to directory ' +
img.basedir)
else:
mylog.info('Overwriting output files (if any) in directory ' +
img.basedir)
if os.path.isdir(img.basedir):
os.system("rm -fr " + img.basedir + '/*')
if not os.path.isdir(img.basedir):
os.makedirs(img.basedir)
# Now add solname (if any) and time to basedir
if img.opts.solnname is not None:
img.basedir += img.opts.solnname + '_'
img.basedir += time.strftime("%d%b%Y_%H.%M.%S")
# Make the final output directory
if not os.path.isdir(img.basedir):
os.makedirs(img.basedir)
del data
img.completed_Ops.append('readimage')
return img
def __call__(self, img):
global bar1
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"SpectIndex")
img.mylog = mylog
if img.opts.spectralindex_do:
mylogger.userinfo(mylog, '\nExtracting spectral indices for all ch0 sources')
shp = img.image_arr.shape
if shp[1] > 1:
# calc freq, beam_spectrum for nchan channels
self.freq_beamsp_unav(img)
sbeam = img.beam_spectrum
freqin = img.freq
# calc initial channel flags if needed
iniflags = self.iniflag(img)
img.specind_iniflags = iniflags
good_chans = N.where(iniflags == False)
unav_image = img.image_arr[0][good_chans]
unav_freqs = freqin[good_chans]
nmax_to_avg = img.opts.specind_maxchan
nchan = unav_image.shape[0]
mylog.info('After initial flagging of channels by rms, %i good channels remain' % (nchan,))
if nmax_to_avg == 0:
nmax_to_avg = nchan
# calculate the rms map of each unflagged channel
bar1 = statusbar.StatusBar('Determing rms for channels in image ..... : ', 0, nchan)
if img.opts.quiet == False:
bar1.start()
rms_spec = self.rms_spectrum(img, unav_image) # bar1 updated here
bar2 = statusbar.StatusBar('Calculating spectral indices for sources : ', 0, img.nsrc)
c_wts = img.opts.collapse_wt
snr_desired = img.opts.specind_snr
if img.opts.quiet == False and img.opts.verbose_fitting == False:
bar2.start()
for src in img.sources:
isl = img.islands[src.island_id]
isl_bbox = isl.bbox
# Fit each channel with ch0 Gaussian(s) of the source,
# allowing only the normalization to vary.
chan_images = unav_image[:, isl_bbox[0], isl_bbox[1]]
chan_rms = rms_spec[:, isl_bbox[0], isl_bbox[1]]
beamlist = img.beam_spectrum
unavg_total_flux, e_unavg_total_flux = self.fit_channels(img, chan_images, chan_rms, src, beamlist)
# Check for upper limits and mask. gaus_mask is array of (N_channels x N_gaussians)
# and is True if measured flux is upper limit. n_good_chan_per_gaus is array of N_gaussians
# that gives number of unmasked channels for each Gaussian.
gaus_mask, n_good_chan_per_gaus = self.mask_upper_limits(unavg_total_flux, e_unavg_total_flux, snr_desired)
# Average if needed and fit again
# First find flux of faintest Gaussian of source and use it to estimate rms_desired
gflux = []
for g in src.gaussians:
gflux.append(g.peak_flux)
rms_desired = min(gflux)/snr_desired
total_flux = unavg_total_flux
e_total_flux = e_unavg_total_flux
freq_av = unav_freqs
nchan = chan_images.shape[0]
nchan_prev = nchan
while min(n_good_chan_per_gaus) < 2 and nchan > 2:
avimages, beamlist, freq_av, crms_av = self.windowaverage_cube(chan_images, rms_desired, chan_rms,
c_wts, sbeam, freqin, nmax_to_avg=nmax_to_avg)
total_flux, e_total_flux = self.fit_channels(img, avimages, crms_av, src, beamlist)
gaus_mask, n_good_chan_per_gaus = self.mask_upper_limits(total_flux, e_total_flux, snr_desired)
nchan = avimages.shape[0]
if nchan == nchan_prev:
break
nchan_prev = nchan
rms_desired *= 0.8
# Now fit Gaussian fluxes to obtain spectral indices.
# Only fit if there are detections (at specified sigma threshold)
# in at least two bands. If not, don't fit and set spec_indx
# and error to NaN.
for ig, gaussian in enumerate(src.gaussians):
npos = len(N.where(total_flux[:, ig] > 0.0)[0])
if img.opts.verbose_fitting:
if img.opts.flagchan_snr:
print 'Gaussian #%i : averaged to %i channels, of which %i meet SNR criterion' % (gaussian.gaus_num,
len(total_flux[:, ig]), n_good_chan_per_gaus[ig])
else:
print 'Gaussian #%i : averaged to %i channels, all of which will be used' % (gaussian.gaus_num,
len(total_flux[:, ig]))
if (img.opts.flagchan_snr and n_good_chan_per_gaus[ig] < 2) or npos < 2:
gaussian.spec_indx = N.NaN
gaussian.e_spec_indx = N.NaN
gaussian.spec_norm = N.NaN
gaussian.specin_flux = [N.NaN]
gaussian.specin_fluxE = [N.NaN]
gaussian.specin_freq = [N.NaN]
gaussian.specin_freq0 = N.NaN
else:
if img.opts.flagchan_snr:
good_fluxes_ind = N.where(gaus_mask[:, ig] == False)
else:
good_fluxes_ind = range(len(freq_av))
fluxes_to_fit = total_flux[:, ig][good_fluxes_ind]
e_fluxes_to_fit = e_total_flux[:, ig][good_fluxes_ind]
freqs_to_fit = freq_av[good_fluxes_ind]
fit_res = self.fit_specindex(freqs_to_fit, fluxes_to_fit, e_fluxes_to_fit)
gaussian.spec_norm, gaussian.spec_indx, gaussian.e_spec_indx = fit_res
gaussian.specin_flux = fluxes_to_fit.tolist()
gaussian.specin_fluxE = e_fluxes_to_fit.tolist()
gaussian.specin_freq = freqs_to_fit.tolist()
gaussian.specin_freq0 = N.median(freqs_to_fit)
# Next fit total source fluxes for spectral index.
if len(src.gaussians) > 1:
# First, check unaveraged SNRs for total source.
src_total_flux = N.zeros((chan_images.shape[0], 1))
src_e_total_flux = N.zeros((chan_images.shape[0], 1))
src_total_flux[:,0] = N.sum(unavg_total_flux, 1) # sum over all Gaussians in source to obtain total fluxes in each channel
src_e_total_flux[:,0] = N.sqrt(N.sum(N.power(e_unavg_total_flux, 2.0), 1))
src_mask, n_good_chan = self.mask_upper_limits(src_total_flux, src_e_total_flux, snr_desired)
# Average if needed and fit again
rms_desired = src.peak_flux_max/snr_desired
total_flux = unavg_total_flux
e_total_flux = e_unavg_total_flux
freq_av = unav_freqs
nchan = chan_images.shape[0]
nchan_prev = nchan
while n_good_chan < 2 and nchan > 2:
avimages, beamlist, freq_av, crms_av = self.windowaverage_cube(chan_images, rms_desired, chan_rms,
c_wts, sbeam, freqin, nmax_to_avg=nmax_to_avg)
total_flux, e_total_flux = self.fit_channels(img, avimages, crms_av, src, beamlist)
src_total_flux = N.sum(total_flux, 1) # sum over all Gaussians in source to obtain total fluxes in each channel
src_e_total_flux = N.sqrt(N.sum(N.power(e_total_flux, 2.0), 1))
src_mask, n_good_chan = self.mask_upper_limits(src_total_flux, src_e_total_flux, snr_desired)
nchan = avimages.shape[0]
if nchan == nchan_prev:
break
nchan_prev = nchan
rms_desired *= 0.8
# Now fit source for spectral index.
src_total_flux = src_total_flux.reshape((src_total_flux.shape[0],))
src_e_total_flux = src_e_total_flux.reshape((src_e_total_flux.shape[0],))
src_mask = src_mask.reshape((src_mask.shape[0],))
if img.opts.verbose_fitting:
if img.opts.flagchan_snr:
print 'Source #%i : averaged to %i channels, of which %i meet SNR criterion' % (src.source_id,
len(src_total_flux), nchan)
else:
print 'Source #%i : averaged to %i channels, all of which will be used' % (src.source_id,
len(src_total_flux))
npos = len(N.where(src_total_flux > 0.0)[0])
if isinstance(n_good_chan, int):
n_good_chan = [n_good_chan]
if (img.opts.flagchan_snr and n_good_chan[0] < 2) or npos < 2:
src.spec_indx = N.NaN
src.e_spec_indx = N.NaN
src.spec_norm = N.NaN
src.specin_flux = [N.NaN]
src.specin_fluxE = [N.NaN]
src.specin_freq = [N.NaN]
src.specin_freq0 = N.NaN
else:
if img.opts.flagchan_snr:
good_fluxes_ind = N.where(src_mask == False)
else:
good_fluxes_ind = range(len(freq_av))
fluxes_to_fit = src_total_flux[good_fluxes_ind]
e_fluxes_to_fit = src_e_total_flux[good_fluxes_ind]
freqs_to_fit = freq_av[good_fluxes_ind]
# if len(freqs_to_fit.shape) == 2:
# freqs_to_fit = freqs_to_fit.reshape((freqs_to_fit.shape[0],))
# if len(fluxes_to_fit.shape) == 2:
# fluxes_to_fit = fluxes_to_fit.reshape((fluxes_to_fit.shape[0],))
# if len(e_fluxes_to_fit.shape) == 2:
# e_fluxes_to_fit = e_fluxes_to_fit.reshape((e_fluxes_to_fit.shape[0],))
fit_res = self.fit_specindex(freqs_to_fit, fluxes_to_fit, e_fluxes_to_fit)
src.spec_norm, src.spec_indx, src.e_spec_indx = fit_res
src.specin_flux = fluxes_to_fit.tolist()
src.specin_fluxE = e_fluxes_to_fit.tolist()
src.specin_freq = freqs_to_fit.tolist()
src.specin_freq0 = N.median(freqs_to_fit)
else:
src.spec_norm = src.gaussians[0].spec_norm
src.spec_indx = src.gaussians[0].spec_indx
src.e_spec_indx = src.gaussians[0].e_spec_indx
src.specin_flux = src.gaussians[0].specin_flux
src.specin_fluxE = src.gaussians[0].specin_fluxE
src.specin_freq = src.gaussians[0].specin_freq
src.specin_freq0 = src.gaussians[0].specin_freq0
if bar2.started:
bar2.increment()
img.completed_Ops.append('spectralindex')
else:
mylog.warning('Image has only one channel. Spectral index module disabled.')
img.opts.spectralindex_do = False
def __call__(self, img):
mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Preprocess")
bstat = func.bstat
if img.opts.kappa_clip is None:
kappa = -img.pixel_beamarea()
else:
kappa = img.opts.kappa_clip
if img.opts.polarisation_do:
pols = ['I', 'Q', 'U', 'V']
ch0images = [
img.ch0_arr, img.ch0_Q_arr, img.ch0_U_arr, img.ch0_V_arr
]
img.clipped_mean_QUV = []
img.clipped_rms_QUV = []
else:
pols = ['I'] # assume I is always present
ch0images = [img.ch0_arr]
if hasattr(img, 'rms_mask'):
mask = img.rms_mask
else:
mask = img.mask_arr
opts = img.opts
for ipol, pol in enumerate(pols):
image = ch0images[ipol]
### basic stats
mean, rms, cmean, crms, cnt = bstat(image, mask, kappa)
if cnt > 198:
cmean = mean
crms = rms
if pol == 'I':
if func.approx_equal(crms, 0.0, rel=None):
raise RuntimeError('Clipped rms appears to be zero. Check for regions '\
'with values of 0 and\nblank them (with NaNs) '\
'or use trim_box to exclude them.')
img.raw_mean = mean
img.raw_rms = rms
img.clipped_mean = cmean
img.clipped_rms = crms
mylog.info('%s %.4f %s %.4f %s ' % ("Raw mean (Stokes I) = ", mean*1000.0, \
'mJy and raw rms = ',rms*1000.0, 'mJy'))
mylog.info('%s %.4f %s %s %.4f %s ' % ("sigma clipped mean (Stokes I) = ", cmean*1000.0, \
'mJy and ','sigma clipped rms = ',crms*1000.0, 'mJy'))
else:
img.clipped_mean_QUV.append(cmean)
img.clipped_rms_QUV.append(crms)
mylog.info('%s %s %s %.4f %s %s %.4f %s ' % ("sigma clipped mean (Stokes ", pol, ") = ", cmean*1000.0, \
'mJy and ','sigma clipped rms = ',crms*1000.0, 'mJy'))
image = img.ch0_arr
# Check if pixels are outside the universe
if opts.check_outsideuniv:
mylogger.userinfo(mylog,
"Checking for pixels outside the universe")
noutside_univ = self.outside_univ(img)
img.noutside_univ = noutside_univ
frac_blank = round(
float(noutside_univ) / float(image.shape[0] * image.shape[1]),
3)
mylogger.userinfo(
mylog, "Number of additional pixels blanked",
str(noutside_univ) + ' (' + str(frac_blank * 100.0) + '%)')
else:
noutside_univ = 0
# If needed, (re)mask the image
if noutside_univ > 0:
mask = N.isnan(img.ch0_arr)
masked = mask.any()
img.masked = masked
if masked:
img.mask_arr = mask
img.blankpix = N.sum(mask)
### max/min pixel value & coordinates
shape = image.shape[0:2]
if mask != None:
img.blankpix = N.sum(mask)
if img.blankpix == 0:
max_idx = image.argmax()
min_idx = image.argmin()
else:
max_idx = N.nanargmax(image)
min_idx = N.nanargmin(image)
img.maxpix_coord = N.unravel_index(max_idx, shape)
img.minpix_coord = N.unravel_index(min_idx, shape)
img.max_value = image.flat[max_idx]
img.min_value = image.flat[min_idx]
### Solid angle of the image
cdelt = N.array(img.wcs_obj.acdelt[:2])
img.omega = N.product(shape) * abs(
N.product(cdelt)) / (180. * 180. / pi / pi)
### Total flux in ch0 image
if 'atrous' in img.filename or img._pi or img.log == 'Detection image':
# Don't do this estimate for atrous wavelet images
# or polarized intensity image,
# as it doesn't give the correct flux. Also, ignore
# the flux in the detection image, as it's likely
# wrong (e.g., not corrected for the primary beam).
img.ch0_sum_jy = 0
else:
im_flux = N.nansum(image) / img.pixel_beamarea() # Jy
img.ch0_sum_jy = im_flux
mylogger.userinfo(mylog, 'Flux from sum of (non-blank) pixels',
'%.3f Jy' % (im_flux, ))
### if image seems confused, then take background mean as zero instead
alpha_sourcecounts = 2.5 # approx diff src count slope. 2.2?
if opts.bmpersrc_th is None:
n = (image >= 5. * crms).sum()
if n <= 0:
n = 1
mylog.info('No pixels in image > 5-sigma.')
mylog.info('Taking number of pixels above 5-sigma as 1.')
img.bmpersrc_th = N.product(shape) / (
(alpha_sourcecounts - 1.) * n)
mylog.info('%s %6.2f' %
('Estimated bmpersrc_th = ', img.bmpersrc_th))
else:
img.bmpersrc_th = opts.bmpersrc_th
mylog.info('%s %6.2f' %
('Taking default bmpersrc_th = ', img.bmpersrc_th))
confused = False
if opts.mean_map == 'default':
if opts.bmpersrc_th <= 25. or cmean / crms >= 0.1:
confused = True
img.confused = confused
mylog.info('Parameter confused is ' + str(img.confused))
img.completed_Ops.append('preprocess')
return img
def init_beam(self, img):
"""Initialize beam parameters, and conversion routines
to convert beam to/from pixel coordinates"""
from const import fwsig
mylog = mylogger.logging.getLogger("PyBDSM.InitBeam")
hdr = img.header
cdelt1, cdelt2 = img.wcs_obj.acdelt[0:2]
### define beam conversion routines:
def beam2pix(x):
""" Converts beam in deg to pixels. Use when no dependence on
position is appropriate.
Input beam angle should be degrees CCW from North at image center.
The output beam angle is degrees CCW from the +y axis of the image.
"""
bmaj, bmin, bpa = x
s1 = abs(bmaj / cdelt1)
s2 = abs(bmin / cdelt2)
th = bpa
return (s1, s2, th)
def pix2beam(x):
""" Converts beam in pixels to deg. Use when no dependence on
position is appropriate.
Input beam angle should be degrees CCW from the +y axis of the image.
The output beam angle is degrees CCW from North at image center.
"""
s1, s2, th = x
bmaj = abs(s1 * cdelt1)
bmin = abs(s2 * cdelt2)
bpa = th
if bmaj < bmin:
bmaj, bmin = bmin, bmaj
bpa += 90.0
bpa = divmod(bpa, 180)[1] ### bpa lies between 0 and 180
return [bmaj, bmin, bpa]
def pixel_beam():
"""Returns the beam in sigma units in pixels"""
pbeam = beam2pix(img.beam)
return (pbeam[0] / fwsig, pbeam[1] / fwsig, pbeam[2])
def pixel_beamarea():
"""Returns the beam area in pixels"""
pbeam = beam2pix(img.beam)
return 1.1331 * pbeam[0] * pbeam[1]
### Get the beam information from the header
found = False
if img.opts.beam is not None:
beam = img.opts.beam
else:
try:
beam = (hdr['BMAJ'], hdr['BMIN'], hdr['BPA'])
found = True
except:
### try see if AIPS as put the beam in HISTORY as usual
for h in hdr['HISTORY']:
# Check if h is a string or a FITS Card object (long headers are
# split into Cards as of PyFITS 3.0.4)
if not isinstance(h, str):
hstr = h.value
else:
hstr = h
if N.all([
'BMAJ' in hstr, 'BMIN' in hstr, 'BPA' in hstr,
'CLEAN' in hstr
]):
try:
dum, dum, dum, bmaj, dum, bmin, dum, bpa = hstr.split(
)
except ValueError:
try:
dum, dum, bmaj, dum, bmin, dum, bpa, dum, dum = hstr.split(
)
except ValueError:
break
beam = (float(bmaj), float(bmin), float(bpa))
found = True
if not found:
raise RuntimeError(
"No beam information found in image header.")
### convert beam into pixels (at image center)
pbeam = beam2pix(beam)
pbeam = (pbeam[0] / fwsig, pbeam[1] / fwsig, pbeam[2]
) # IN SIGMA UNITS
### and store it
img.pix2beam = pix2beam
img.beam2pix = beam2pix
img.beam = beam # FWHM size in degrees
img.pixel_beam = pixel_beam # IN SIGMA UNITS in pixels
img.pixel_beamarea = pixel_beamarea
mylogger.userinfo(
mylog, 'Beam shape (major, minor, pos angle)',
'(%.5e, %.5e, %s) degrees' % (beam[0], beam[1], round(beam[2], 1)))
def calculate_maps(self,
img,
data,
mean,
rms,
mask,
map_opts,
do_adapt,
bright_pt_coords=[],
rms_box2=None,
logname=None,
ncores=None):
"""Calls map_2d and checks for problems"""
mylog = mylogger.logging.getLogger("PyBDSM." + img.log +
"Rmsimage.Calcmaps ")
rms_ok = False
mylog = mylogger.logging.getLogger("PyBDSM." + img.log +
"Rmsimage.Calcmaps ")
opts = img.opts
kappa = map_opts[0]
while not rms_ok:
self.map_2d(data,
mean,
rms,
mask,
*map_opts,
do_adapt=do_adapt,
bright_pt_coords=bright_pt_coords,
rms_box2=rms_box2,
logname=logname,
ncores=ncores)
if N.any(rms < 0.0):
rms_ok = False
if (opts.rms_box_bright is None
and do_adapt) or (opts.rms_box is None
and not do_adapt):
# Increase box by 20%
if do_adapt:
new_width = int(img.rms_box_bright[0] * 1.2)
if new_width == img.rms_box_bright[0]:
new_width = img.rms_box_bright[0] + 1
new_step = int(new_width / 3.0)
img.rms_box_bright = (new_width, new_step)
if img.rms_box_bright[0] > min(
img.ch0_arr.shape) / 4.0:
mylogger.userinfo(
mylog,
'Size of rms_box_bright larger than 1/4 of image size'
)
mylogger.userinfo(
mylog,
'Using constant background rms and mean')
img.use_rms_map = False
img.rms_box = img.rms_box_bright
img.mean_map_type = 'const'
rms_ok = True
else:
map_opts = (kappa, img.rms_box_bright,
opts.spline_rank)
else:
new_width = int(img.rms_box[0] * 1.2)
if new_width == img.rms_box[0]:
new_width = img.rms_box[0] + 1
new_step = int(new_width / 3.0)
img.rms_box = (new_width, new_step)
if img.rms_box[0] > min(img.ch0_arr.shape) / 4.0:
mylogger.userinfo(
mylog,
'Size of rms_box larger than 1/4 of image size'
)
mylogger.userinfo(
mylog,
'Using constant background rms and mean')
img.use_rms_map = False
img.mean_map_type = 'const'
rms_ok = True
else:
map_opts = (kappa, img.rms_box, opts.spline_rank)
else:
# User has specified box size, use order=1 to prevent negatives
if opts.spline_rank > 1:
mylog.warning(
'Negative values found in rms map interpolated with spline_rank = %i'
% opts.spline_rank)
mylog.warning(
'Using spline_rank = 1 (bilinear interpolation) instead'
)
if do_adapt:
map_opts = (kappa, img.rms_box_bright, 1)
else:
map_opts = (kappa, img.rms_box, 1)
else:
rms_ok = True
return mean, rms
def __call__(self, img):
mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Islands")
opts = img.opts
minsize = opts.minpix_isl
if minsize is None:
minsize = int(img.pixel_beamarea() /
3.0) # 1/3 of beam area in pixels
if minsize < 6:
minsize = 6 # Need at least 6 pixels to obtain good fits
mylogger.userinfo(mylog, "Minimum number of pixels per island",
'%i' % minsize)
img.minpix_isl = minsize
if opts.detection_image != '':
# Use a different image for island detection. The detection
# image and the measurement image must have the same shape
# and be registered. Otherwise, one could reproject the
# detection image using, e.g., the Kapteyn package.
#
# First, set up up an Image object and run a limited
# op_chain.
from . import _run_op_list
mylogger.userinfo(mylog,
"\nDetermining islands from detection image")
det_chain, det_opts = self.setpara_bdsm(img, opts.detection_image)
det_img = Image(det_opts)
det_img.log = 'Detection image'
success = _run_op_list(det_img, det_chain)
if not success:
return
# Check that the ch0 images are the same size
ch0_map = img.ch0_arr
det_ch0_map = det_img.ch0_arr
det_shape = det_ch0_map.shape
ch0_shape = ch0_map.shape
if det_shape != ch0_shape:
raise RuntimeError(
"Detection image shape does not match that of input image."
)
# Run through islands and correct the image and rms, mean and max values
img.island_labels = det_img.island_labels
corr_islands = []
mean_map = img.mean_arr
rms_map = img.rms_arr
for i, isl in enumerate(det_img.islands):
islcp = isl.copy(img.pixel_beamarea(),
image=ch0_map[isl.bbox],
mean=mean_map[isl.bbox],
rms=rms_map[isl.bbox])
islcp.island_id = i
corr_islands.append(islcp)
img.islands = corr_islands
img.nisl = len(img.islands)
img.pyrank = det_img.pyrank
img.minpix_isl = det_img.minpix_isl
mylogger.userinfo(mylog,
"\nContinuing processing using primary image")
else:
if opts.src_ra_dec is not None:
mylogger.userinfo(
mylog,
"Constructing islands at user-supplied source locations")
img.islands = self.coords_to_isl(img, opts)
else:
img.islands = self.ndimage_alg(img, opts)
img.nisl = len(img.islands)
mylogger.userinfo(mylog, "Number of islands found",
'%i' % len(img.islands))
ch0_map = img.ch0_arr
ch0_shape = ch0_map.shape
pyrank = N.zeros(ch0_shape, dtype=N.int32)
for i, isl in enumerate(img.islands):
isl.island_id = i
pyrank[isl.bbox] += N.invert(isl.mask_active) * (i + 1)
pyrank -= 1 # align pyrank values with island ids and set regions outside of islands to -1
if opts.output_all: write_islands(img)
if opts.savefits_rankim:
func.write_image_to_file(img.use_io,
img.imagename + '_pyrank.fits',
pyrank, img)
img.pyrank = pyrank
img.completed_Ops.append('islands')
return img
def __call__(self, img):
mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Polarisatn")
if img.opts.polarisation_do:
mylog.info('Extracting polarisation properties for all sources')
pols = ['I', 'Q', 'U', 'V']
# Run gausfit and gual2srl on PI image to look for polarized sources
# undetected in I
fit_PI = img.opts.pi_fit
n_new = 0
ch0_pi = N.sqrt(img.ch0_Q_arr**2 + img.ch0_U_arr**2)
img.ch0_pi_arr = ch0_pi
if fit_PI:
from . import _run_op_list
mylogger.userinfo(mylog, "\nChecking PI image for new sources")
mask = img.mask_arr
minsize = img.opts.minpix_isl
# Set up image object for PI image.
pi_chain, pi_opts = self.setpara_bdsm(img)
pimg = Image(pi_opts)
pimg.beam = img.beam
pimg.pixel_beam = img.pixel_beam
pimg.pixel_beamarea = img.pixel_beamarea
pimg.log = 'PI.'
pimg.pix2beam = img.pix2beam
pimg.beam2pix = img.beam2pix
pimg.pix2gaus = img.pix2gaus
pimg.gaus2pix = img.gaus2pix
pimg.pix2sky = img.pix2sky
pimg.sky2pix = img.sky2pix
pimg.pix2coord = img.pix2coord
pimg.wcs_obj = img.wcs_obj
pimg.mask_arr = mask
pimg.masked = img.masked
pimg.ch0_arr = ch0_pi
pimg._pi = True
success = _run_op_list(pimg, pi_chain)
if not success:
return
img.pi_islands = pimg.islands
img.pi_gaussians = pimg.gaussians
img.pi_sources = pimg.sources
# Now check for new sources in the PI image that are not
# found in the Stokes I image. If any new sources are found,
# adjust their IDs to follow after those found in I.
new_isl = []
new_src = []
new_gaus = []
n_new_src = 0
isl_id = img.islands[-1].island_id
src_id = img.sources[-1].source_id
gaus_id = img.gaussians[-1].gaus_num
for pi_isl in pimg.islands:
new_sources = []
for pi_src in pi_isl.sources:
if img.pyrank[int(img.sky2pix(pi_src.posn_sky_max)[0]),
int(img.sky2pix(pi_src.posn_sky_max)[1])] == -1:
src_id += 1
pi_src._pi = True
pi_src.island_id = isl_id
pi_src.source_id = src_id
pi_src.spec_indx = N.NaN
pi_src.e_spec_indx = N.NaN
pi_src.spec_norm = N.NaN
pi_src.specin_flux = [N.NaN]
pi_src.specin_fluxE = [N.NaN]
pi_src.specin_freq = [N.NaN]
pi_src.specin_freq0 = N.NaN
new_sources.append(pi_src)
new_src.append(pi_src)
n_new_src += 1
for g in pi_src.gaussians:
gaus_id += 1
new_gaus.append(g)
g.gaus_num = gaus_id
if len(new_sources) > 0:
isl_id += 1
pi_isl.sources = new_sources
pi_isl.island_id = isl_id
pi_isl._pi = True
new_isl.append(pi_isl)
n_new = len(new_isl)
mylogger.userinfo(mylog, "New sources found in PI image", '%i (%i total)' %
(n_new, img.nsrc+n_new))
if n_new > 0:
img.islands += new_isl
img.sources += new_src
img.gaussians += new_gaus
img.nsrc += n_new_src
bar = statusbar.StatusBar('Calculating polarisation properties .... : ', 0, img.nsrc)
if img.opts.quiet == False:
bar.start()
for isl in img.islands:
isl_bbox = isl.bbox
ch0_I = img.ch0_arr[isl_bbox]
ch0_Q = img.ch0_Q_arr[isl_bbox]
ch0_U = img.ch0_U_arr[isl_bbox]
ch0_V = img.ch0_V_arr[isl_bbox]
ch0_images = [ch0_I, ch0_Q, ch0_U, ch0_V]
for i, src in enumerate(isl.sources):
# For each source, assume the morphology does not change
# across the Stokes cube. This assumption allows us to fit
# the Gaussians of each source to each Stokes image by
# simply fitting only the overall normalizations of the
# individual Gaussians.
#
# First, fit all source Gaussians to each Stokes image:
x, y = N.mgrid[isl_bbox]
gg = src.gaussians
fitfix = N.ones(len(gg)) # fit only normalization
srcmask = isl.mask_active
total_flux = N.zeros((4, len(fitfix)), dtype=N.float32) # array of fluxes: N_Stokes x N_Gaussians
errors = N.zeros((4, len(fitfix)), dtype=N.float32) # array of fluxes: N_Stokes x N_Gaussians
for sind, image in enumerate(ch0_images):
if (sind==0 and hasattr(src, '_pi')) or sind > 0: # Fit I only for PI sources
p, ep = func.fit_mulgaus2d(image, gg, x, y, srcmask, fitfix)
for ig in range(len(fitfix)):
center_pix = (p[ig*6 + 1], p[ig*6 + 2])
bm_pix = N.array([img.pixel_beam()[0], img.pixel_beam()[1], img.pixel_beam()[2]])
total_flux[sind, ig] = p[ig*6]*p[ig*6+3]*p[ig*6+4]/(bm_pix[0]*bm_pix[1])
p = N.insert(p, N.arange(len(fitfix))*6+6, total_flux[sind])
if sind > 0:
rms_img = img.__getattribute__('rms_'+pols[sind]+'_arr')
else:
rms_img = img.rms_arr
if len(rms_img.shape) > 1:
rms_isl = rms_img[isl.bbox].mean()
else:
rms_isl = rms_img
errors[sind] = func.get_errors(img, p, rms_isl)[6]
# Now, assign fluxes to each Gaussian.
src_flux_I = 0.0
src_flux_Q = 0.0
src_flux_U = 0.0
src_flux_V = 0.0
src_flux_I_err_sq = 0.0
src_flux_Q_err_sq = 0.0
src_flux_U_err_sq = 0.0
src_flux_V_err_sq = 0.0
for ig, gaussian in enumerate(src.gaussians):
flux_I = total_flux[0, ig]
flux_I_err = abs(errors[0, ig])
flux_Q = total_flux[1, ig]
flux_Q_err = abs(errors[1, ig])
flux_U = total_flux[2, ig]
flux_U_err = abs(errors[2, ig])
flux_V = total_flux[3, ig]
flux_V_err = abs(errors[3, ig])
if hasattr(src, '_pi'):
gaussian.total_flux = flux_I
gaussian.total_fluxE = flux_I_err
gaussian.total_flux_Q = flux_Q
gaussian.total_flux_U = flux_U
gaussian.total_flux_V = flux_V
gaussian.total_fluxE_Q = flux_Q_err
gaussian.total_fluxE_U = flux_U_err
gaussian.total_fluxE_V = flux_V_err
if hasattr(src, '_pi'):
src_flux_I += flux_I
src_flux_I_err_sq += flux_I_err**2
src_flux_Q += flux_Q
src_flux_U += flux_U
src_flux_V += flux_V
src_flux_Q_err_sq += flux_Q_err**2
src_flux_U_err_sq += flux_U_err**2
src_flux_V_err_sq += flux_V_err**2
# Calculate and store polarisation fractions and angle for each Gaussian in the island
# For this we need the I flux, which we can just take from g.total_flux and src.total_flux
flux_I = gaussian.total_flux
flux_I_err = gaussian.total_fluxE
stokes = [flux_I, flux_Q, flux_U, flux_V]
stokes_err = [flux_I_err, flux_Q_err, flux_U_err, flux_V_err]
lpol_frac, lpol_frac_loerr, lpol_frac_hierr = self.calc_lpol_fraction(stokes, stokes_err) # linear pol fraction
lpol_ang, lpol_ang_err = self.calc_lpol_angle(stokes, stokes_err) # linear pol angle
cpol_frac, cpol_frac_loerr, cpol_frac_hierr = self.calc_cpol_fraction(stokes, stokes_err) # circular pol fraction
tpol_frac, tpol_frac_loerr, tpol_frac_hierr = self.calc_tpol_fraction(stokes, stokes_err) # total pol fraction
gaussian.lpol_fraction = lpol_frac
gaussian.lpol_fraction_loerr = lpol_frac_loerr
gaussian.lpol_fraction_hierr = lpol_frac_hierr
gaussian.cpol_fraction = cpol_frac
gaussian.cpol_fraction_loerr = cpol_frac_loerr
gaussian.cpol_fraction_hierr = cpol_frac_hierr
gaussian.tpol_fraction = tpol_frac
gaussian.tpol_fraction_loerr = tpol_frac_loerr
gaussian.tpol_fraction_hierr = tpol_frac_hierr
gaussian.lpol_angle = lpol_ang
gaussian.lpol_angle_err = lpol_ang_err
# Store fluxes for each source in the island
if hasattr(src, '_pi'):
src.total_flux = src_flux_I
src.total_fluxE = N.sqrt(src_flux_I_err_sq)
src.total_flux_Q = src_flux_Q
src.total_flux_U = src_flux_U
src.total_flux_V = src_flux_V
src.total_fluxE_Q = N.sqrt(src_flux_Q_err_sq)
src.total_fluxE_U = N.sqrt(src_flux_U_err_sq)
src.total_fluxE_V = N.sqrt(src_flux_V_err_sq)
# Calculate and store polarisation fractions and angle for each source in the island
# For this we need the I flux, which we can just take from g.total_flux and src.total_flux
src_flux_I = src.total_flux
src_flux_I_err = src.total_fluxE
stokes = [src_flux_I, src_flux_Q, src_flux_U, src_flux_V]
stokes_err = [src_flux_I_err, N.sqrt(src_flux_Q_err_sq), N.sqrt(src_flux_U_err_sq), N.sqrt(src_flux_V_err_sq)]
lpol_frac, lpol_frac_loerr, lpol_frac_hierr = self.calc_lpol_fraction(stokes, stokes_err) # linear pol fraction
lpol_ang, lpol_ang_err = self.calc_lpol_angle(stokes, stokes_err) # linear pol angle
cpol_frac, cpol_frac_loerr, cpol_frac_hierr = self.calc_cpol_fraction(stokes, stokes_err) # circular pol fraction
tpol_frac, tpol_frac_loerr, tpol_frac_hierr = self.calc_tpol_fraction(stokes, stokes_err) # total pol fraction
src.lpol_fraction = lpol_frac
src.lpol_fraction_loerr = lpol_frac_loerr
src.lpol_fraction_hierr = lpol_frac_hierr
src.cpol_fraction = cpol_frac
src.cpol_fraction_loerr = cpol_frac_loerr
src.cpol_fraction_hierr = cpol_frac_hierr
src.tpol_fraction = tpol_frac
src.tpol_fraction_loerr = tpol_frac_loerr
src.tpol_fraction_hierr = tpol_frac_hierr
src.lpol_angle = lpol_ang
src.lpol_angle_err = lpol_ang_err
if bar.started:
bar.increment()
bar.stop()
img.completed_Ops.append('polarisation')