Exemple #1
0
    def check_rmsmap(self, img, rms):
        """Calculates the statistics of the rms map and decides, when
        rms_map=None, whether to take the map (if variance
        is significant) or a constant value
        """
        from math import sqrt

        mylog = mylogger.logging.getLogger("PyBDSM." + img.log +
                                           "Rmsimage.Checkrms  ")
        cdelt = img.wcs_obj.acdelt[:2]
        bm = (img.beam[0], img.beam[1])
        fw_pix = sqrt(N.product(bm) / abs(N.product(cdelt)))
        if img.masked:
            unmasked = N.where(~img.mask_arr)
            stdsub = N.std(rms[unmasked])
            maxrms = N.max(rms[unmasked])
        else:
            stdsub = N.std(rms)
            maxrms = N.max(rms)

        rms_expect = img.clipped_rms / sqrt(2) / img.rms_box[0] * fw_pix
        mylog.debug(
            '%s %10.6f %s' %
            ('Standard deviation of rms image = ', stdsub * 1000.0, 'mJy'))
        mylog.debug(
            '%s %10.6f %s' %
            ('Expected standard deviation = ', rms_expect * 1000.0, 'mJy'))
        if stdsub > 1.1 * rms_expect:
            img.use_rms_map = True
            mylogger.userinfo(mylog, 'Variation in rms image significant')
        else:
            img.use_rms_map = False
            mylogger.userinfo(mylog, 'Variation in rms image not significant')

        return img
Exemple #2
0
 def run_dist_com_all(self, task, wdir, command, SBs=[], nodes='', group=False):
     """Run a run_dist_com on all nodes and all SBs unless specified a restriction.
     If no restriction are specified, run the command for each node and each SB.
     """
     s = self.get_status()
     if s == None:
         self.mylog.error("Not connected to a cluster.")
         return False
     njobs = 0
     for node in s:
         # shall we proceed with this node?
         if (node in nodes or nodes == ''):
             if group:
                 nodeSBs = s[node]['group']
             else:
                 nodeSBs = s[node]['sb']
             for SB in nodeSBs:
                 # shall we proceed with this SB?
                 if (int(SB) in SBs or SBs == []):
                     rcommand = command.replace('$SB', SB)
                     rwdir = wdir.replace('$SB', SB)
                     # for groups rename the variable
                     if group: SB = 'g'+SB
                     self.run_dist_com(task, rwdir, rcommand, SB, node)
                     njobs += 1
     mylogger.userinfo(self.mylog, "Launched "+str(njobs)+" jobs.")
Exemple #3
0
    def check_meanmap(self, img, mean):
        """Calculates the statistics of the mean map and decides, when
        mean_map=None, whether to take the map (if variance
        is significant) or a constant value
        """
    	from math import sqrt

        mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Rmsimage.Checkmean ")
        cdelt = img.wcs_obj.acdelt[:2]
        bm = (img.beam[0], img.beam[1])
        fw_pix = sqrt(N.product(bm)/abs(N.product(cdelt)))
    	if img.masked:
            unmasked = N.where(~img.mask_arr)
            stdsub = N.std(mean[unmasked])
            maxmean = N.max(mean[unmasked])
        else:
            stdsub = N.std(mean)
            maxmean = N.max(mean)
        rms_expect = img.clipped_rms/img.rms_box[0]*fw_pix
        mylog.debug('%s %10.6f %s' % ('Standard deviation of mean image = ', stdsub*1000.0, 'mJy'))
        mylog.debug('%s %10.6f %s' % ('Expected standard deviation = ', rms_expect*1000.0, 'mJy'))

        # For mean map, use a higher threshold than for the rms map, as radio images
        # should rarely, if ever, have significant variations in the mean
        if stdsub > 5.0*rms_expect:
          img.mean_map_type = 'map'
          mylogger.userinfo(mylog, 'Variation in mean image significant')
        else:
          if img.confused:
            img.mean_map_type = 'zero'
          else:
            img.mean_map_type = 'const'
          mylogger.userinfo(mylog, 'Variation in mean image not significant')

        return img
Exemple #4
0
    def check_rmsmap(self, img, rms):
        """Calculates the statistics of the rms map and decides, when
        rms_map=None, whether to take the map (if variance
        is significant) or a constant value
        """
    	from math import sqrt

        mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Rmsimage.Checkrms  ")
        cdelt = img.wcs_obj.acdelt[:2]
    	bm = (img.beam[0], img.beam[1])
    	fw_pix = sqrt(N.product(bm)/abs(N.product(cdelt)))
    	if img.masked:
    	    unmasked = N.where(~img.mask_arr)
            stdsub = N.std(rms[unmasked])
            maxrms = N.max(rms[unmasked])
        else:
            stdsub = N.std(rms)
            maxrms = N.max(rms)

    	rms_expect = img.clipped_rms/sqrt(2)/img.rms_box[0]*fw_pix
        mylog.debug('%s %10.6f %s' % ('Standard deviation of rms image = ', stdsub*1000.0, 'mJy'))
        mylog.debug('%s %10.6f %s' % ('Expected standard deviation = ', rms_expect*1000.0, 'mJy'))
    	if stdsub > 1.1*rms_expect:
            img.use_rms_map = True
            mylogger.userinfo(mylog, 'Variation in rms image significant')
        else:
            img.use_rms_map = False
            mylogger.userinfo(mylog, 'Variation in rms image not significant')

        return img
Exemple #5
0
    def check_meanmap(self, img, mean):
        """Calculates the statistics of the mean map and decides, when
        mean_map=None, whether to take the map (if variance
        is significant) or a constant value
        """
    	from math import sqrt

        mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Rmsimage.Checkmean ")
        cdelt = img.wcs_obj.acdelt[:2]
        bm = (img.beam[0], img.beam[1])
        fw_pix = sqrt(N.product(bm)/abs(N.product(cdelt)))
    	if img.masked:
            unmasked = N.where(~img.mask_arr)
            stdsub = N.std(mean[unmasked])
            maxmean = N.max(mean[unmasked])
        else:
            stdsub = N.std(mean)
            maxmean = N.max(mean)
        rms_expect = img.clipped_rms/img.rms_box[0]*fw_pix
        mylog.debug('%s %10.6f %s' % ('Standard deviation of mean image = ', stdsub*1000.0, 'mJy'))
        mylog.debug('%s %10.6f %s' % ('Expected standard deviation = ', rms_expect*1000.0, 'mJy'))

        # For mean map, use a higher threshold than for the rms map, as radio images
        # should rarely, if ever, have significant variations in the mean
        if stdsub > 5.0*rms_expect:
          img.mean_map_type = 'map'
          mylogger.userinfo(mylog, 'Variation in mean image significant')
        else:
          if img.confused:
            img.mean_map_type = 'zero'
          else:
            img.mean_map_type = 'const'
          mylogger.userinfo(mylog, 'Variation in mean image not significant')

        return img
Exemple #6
0
 def run_dist_com_all(self,
                      task,
                      wdir,
                      command,
                      SBs=[],
                      nodes='',
                      group=False):
     """Run a run_dist_com on all nodes and all SBs unless specified a restriction.
     If no restriction are specified, run the command for each node and each SB.
     """
     s = self.get_status()
     if s == None:
         self.mylog.error("Not connected to a cluster.")
         return False
     njobs = 0
     for node in s:
         # shall we proceed with this node?
         if (node in nodes or nodes == ''):
             if group:
                 nodeSBs = s[node]['group']
             else:
                 nodeSBs = s[node]['sb']
             for SB in nodeSBs:
                 # shall we proceed with this SB?
                 if (int(SB) in SBs or SBs == []):
                     rcommand = command.replace('$SB', SB)
                     rwdir = wdir.replace('$SB', SB)
                     # for groups rename the variable
                     if group: SB = 'g' + SB
                     self.run_dist_com(task, rwdir, rcommand, SB, node)
                     njobs += 1
     mylogger.userinfo(self.mylog, "Launched " + str(njobs) + " jobs.")
Exemple #7
0
 def set_client(self):
     """Set current cluster client"""
     from IPython.parallel import Client
     try:
         self._cluster_client = Client(profile='ssh')
         mylogger.userinfo(self.mylog, "Connected to a cluster.")
         return True
     except:
         return False
Exemple #8
0
 def set_client(self):
     """Set current cluster client"""
     from IPython.parallel import Client
     try:
         self._cluster_client = Client(profile='ssh')
         mylogger.userinfo(self.mylog, "Connected to a cluster.")
         return True
     except:
         return False
Exemple #9
0
    def calculate_maps(self, img, data, mean, rms, mask, map_opts, do_adapt,
                       bright_pt_coords=[], rms_box2=None,
                       logname=None, ncores=None):
        """Calls map_2d and checks for problems"""
        mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Rmsimage.Calcmaps ")
        rms_ok = False
        mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Rmsimage.Calcmaps ")
        opts = img.opts
        kappa = map_opts[0]
        while not rms_ok:
            self.map_2d(data, mean, rms, mask, *map_opts, do_adapt=do_adapt,
                        bright_pt_coords=bright_pt_coords, rms_box2=rms_box2,
                        logname=logname, ncores=ncores)
            if N.any(rms < 0.0):
                rms_ok = False
                if (opts.rms_box_bright is None and do_adapt) or (opts.rms_box is None and not do_adapt):
                    # Increase box by 20%
                    if do_adapt:
                        new_width = int(img.rms_box_bright[0]*1.2)
                        if new_width == img.rms_box_bright[0]:
                            new_width = img.rms_box_bright[0] + 1
                        new_step = int(new_width/3.0)
                        img.rms_box_bright = (new_width, new_step)
                        if img.rms_box_bright[0] > min(img.ch0_arr.shape)/4.0:
                            mylogger.userinfo(mylog, 'Size of rms_box_bright larger than 1/4 of image size')
                            mylogger.userinfo(mylog, 'Using constant background rms and mean')
                            img.use_rms_map = False
                            img.rms_box = img.rms_box_bright
                            img.mean_map_type = 'const'
                            rms_ok = True
                        else:
                            map_opts = (kappa, img.rms_box_bright, opts.spline_rank)
                    else:
                        new_width = int(img.rms_box[0]*1.2)
                        if new_width == img.rms_box[0]:
                            new_width = img.rms_box[0] + 1
                        new_step = int(new_width/3.0)
                        img.rms_box = (new_width, new_step)
                        if img.rms_box[0] > min(img.ch0_arr.shape)/4.0:
                            mylogger.userinfo(mylog, 'Size of rms_box larger than 1/4 of image size')
                            mylogger.userinfo(mylog, 'Using constant background rms and mean')
                            img.use_rms_map = False
                            img.mean_map_type = 'const'
                            rms_ok = True
                        else:
                            map_opts = (kappa, img.rms_box, opts.spline_rank)

                else:
                    # User has specified box size, use order=1 to prevent negatives
                    if opts.spline_rank > 1:
                        mylog.warning('Negative values found in rms map interpolated with spline_rank = %i' % opts.spline_rank)
                        mylog.warning('Using spline_rank = 1 (bilinear interpolation) instead')
                        if do_adapt:
                            map_opts = (kappa, img.rms_box_bright, 1)
                        else:
                            map_opts = (kappa, img.rms_box, 1)
            else:
                rms_ok = True

        return mean, rms
Exemple #10
0
    def run_dist_com(self, task, wdir, command, SB='', node=''):
        """ Run a distributed command using information on the _cluster_status.
        The node parameter is required, the SB not (used only for job identification
        and to schedule a job after already submitted jobs on that SB are done).
        """
        s = self.get_status()
        if s == None:
            self.mylog.error("Not connected to a cluster.")
            return False
        if node == '':
            self.mylog.error("Node parameter required to run a command.")
            return False
        # define the function called by engines
        @interactive
        def f(c, node='', SB='', task='', wdir=''):
            import os, subprocess
            if wdir != '' and os.path.isdir(wdir): os.chdir(wdir)
            s = subprocess.Popen(c, shell=True,\
               stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            s.wait()
            (out, err) = s.communicate()
            return {'node':node, 'SB':SB, 'task':task, 'command':c, \
                        'out':out, 'err':err}

        mylogger.userinfo(
            self.mylog, "Launching on node: " + node + " (SB: " + SB +
            ", wdir:" + wdir + ")\n" + command)
        try:
            scheduler = self.get_client().load_balanced_view(s[node]['e'])
            # Exec a job on a SB only if previous jobs on that
            # SB are finished and finished without errors
            time.sleep(0.1)  # TODO: check if this minimize the buffer problem
            after_ids = self.getSBids(SB)
            with scheduler.temp_flags(after=after_ids, retries=3):
                scheduler.apply_async(f,
                                      command,
                                      node=node,
                                      SB=SB,
                                      task=task,
                                      wdir=wdir)
        except Exception, e:
            self.mylog.exception("Cannot launch the command: " + command + \
                  "(Task: " + task + " - SB: " + SB + " - Node: " + node + ")")
Exemple #11
0
    def run_dist_com(self, task, wdir, command, SB='', node=''):
        """ Run a distributed command using information on the _cluster_status.
        The node parameter is required, the SB not (used only for job identification
        and to schedule a job after already submitted jobs on that SB are done).
        """
        s = self.get_status()
        if s == None:
            self.mylog.error("Not connected to a cluster.")
            return False
        if node == '':
            self.mylog.error("Node parameter required to run a command.")
            return False
        # define the function called by engines
        @interactive
        def f(c, node='', SB='', task='', wdir=''):
            import os, subprocess
            if wdir != '' and os.path.isdir(wdir): os.chdir(wdir)
            s = subprocess.Popen(c, shell=True,\
               stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            s.wait()
            (out, err) = s.communicate()
            return {'node':node, 'SB':SB, 'task':task, 'command':c, \
                        'out':out, 'err':err}

        mylogger.userinfo(self.mylog,"Launching on node: "+node+" (SB: "+SB+", wdir:"+wdir+")\n"+command)
        try:
            scheduler = self.get_client().load_balanced_view(s[node]['e'])
            # Exec a job on a SB only if previous jobs on that
            # SB are finished and finished without errors
            time.sleep(0.1) # TODO: check if this minimize the buffer problem
            after_ids = self.getSBids(SB)
            with scheduler.temp_flags(after=after_ids, retries=3):
                scheduler.apply_async(f, command, node=node, SB=SB, task=task, wdir=wdir)
        except Exception, e:
            self.mylog.exception("Cannot launch the command: " + command + \
                  "(Task: " + task + " - SB: " + SB + " - Node: " + node + ")")
Exemple #12
0
 def output_rmsbox_size(self, img):
     """Prints rms/mean box size"""
     opts = img.opts
     do_adapt = opts.adaptive_rms_box
     mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"RMSimage")
     if (opts.rms_map is not False) or (opts.mean_map not in ['zero', 'const']):
       if do_adapt:
           if opts.rms_box_bright is None:
               mylogger.userinfo(mylog, 'Derived rms_box (box size, step size)',
                                 '(' + str(img.rms_box_bright[0]) + ', ' +
                                 str(img.rms_box_bright[1]) + ') pixels (small scale)')
           else:
               mylogger.userinfo(mylog, 'Using user-specified rms_box',
                                 '(' + str(img.rms_box_bright[0]) + ', ' +
                                 str(img.rms_box_bright[1]) + ') pixels (small scale)')
           if opts.rms_box is None:
               mylogger.userinfo(mylog, 'Derived rms_box (box size, step size)',
                                 '(' + str(img.rms_box[0]) + ', ' +
                                 str(img.rms_box[1]) + ') pixels (large scale)')
           else:
               mylogger.userinfo(mylog, 'Using user-specified rms_box',
                                 '(' + str(img.rms_box[0]) + ', ' +
                                 str(img.rms_box[1]) + ') pixels (large scale)')
       else:
           if opts.rms_box is None:
               mylogger.userinfo(mylog, 'Derived rms_box (box size, step size)',
                             '(' + str(img.rms_box[0]) + ', ' +
                             str(img.rms_box[1]) + ') pixels')
           else:
               mylogger.userinfo(mylog, 'Using user-specified rms_box',
                                 '(' + str(img.rms_box[0]) + ', ' +
                                 str(img.rms_box[1]) + ') pixels')
Exemple #13
0
    def __call__(self, img):
        #  for each island, get the gaussians into a list and then send them to process
        #  src_index is source number, starting from 0
        mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Gaul2Srl")
        mylogger.userinfo(mylog, 'Grouping Gaussians into sources')
        img.aperture = img.opts.aperture
        if img.aperture is not None and img.aperture <= 0.0:
            mylog.warn('Specified aperture is <= 0. Skipping aperture fluxes.')
            img.aperture = None

        src_index = -1
        dsrc_index = 0
        sources = []
        dsources = []
        no_gaus_islands = []
        for iisl, isl in enumerate(img.islands):
            isl_sources = []
            isl_dsources = []
            g_list = []
            for g in isl.gaul:
                if g.flag == 0:
                    g_list.append(g)

            if len(g_list) > 0:
              if len(g_list) == 1:
                src_index, source = self.process_single_gaussian(img, g_list, src_index, code = 'S')
                sources.append(source)
                isl_sources.append(source)
              else:
                src_index, source = self.process_CM(img, g_list, isl, src_index)
                sources.extend(source)
                isl_sources.extend(source)
            else:
                if not img.waveletimage:
                    dg = isl.dgaul[0]
                    no_gaus_islands.append((isl.island_id, dg.centre_pix[0], dg.centre_pix[1]))
                    # Put in the dummy Source as the source and use negative IDs
                    g_list = isl.dgaul
                    dsrc_index, dsource = self.process_single_gaussian(img, g_list, dsrc_index, code = 'S')
                    dsources.append(dsource)
                    isl_dsources.append(dsource)

            isl.sources = isl_sources
            isl.dsources = isl_dsources
        img.sources = sources
        img.dsources = dsources
        img.nsrc = src_index + 1
        mylogger.userinfo(mylog, "Number of sources formed from Gaussians",
                          str(img.nsrc))
        if not img.waveletimage and not img._pi and len(no_gaus_islands) > 0 and not img.opts.quiet:
            message = 'All Gaussians were flagged for the following island'
            if len(no_gaus_islands) == 1:
                message += ':\n'
            else:
                message += 's:\n'
            for isl_id in no_gaus_islands:
                message += '    Island #%i (x=%i, y=%i)\n' % isl_id
            if len(no_gaus_islands) == 1:
                message += 'Please check this island. If it is a valid island and\n'
            else:
                message += 'Please check these islands. If they are valid islands and\n'
            if img.opts.atrous_do:
                message += 'should be fit, try adjusting the flagging options (use\n'\
                           'show_fit with "ch0_flagged=True" to see the flagged Gaussians).'
            else:
                message += 'should be fit, try adjusting the flagging options (use\n'\
                           'show_fit with "ch0_flagged=True" to see the flagged Gaussians)\n'\
                           'or enabling the wavelet module (with "atrous_do=True").'
            message += '\nTo include empty islands in output source catalogs, set\n'\
                        'incl_empty=True in the write_catalog task.'
            mylog.warning(message)

        img.completed_Ops.append('gaul2srl')
Exemple #14
0
    def __call__(self, img):

        mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Wavelet")

        if img.opts.atrous_do:
          if img.nisl == 0:
            mylog.warning("No islands found. Skipping wavelet decomposition.")
            img.completed_Ops.append('wavelet_atrous')
            return

          mylog.info("Decomposing gaussian residual image into a-trous wavelets")
          bdir = img.basedir + '/wavelet/'
          if img.opts.output_all:
              if not os.path.isdir(bdir): os.makedirs(bdir)
              if not os.path.isdir(bdir + '/residual/'): os.makedirs(bdir + '/residual/')
              if not os.path.isdir(bdir + '/model/'): os.makedirs(bdir + '/model/')
          dobdsm = img.opts.atrous_bdsm_do
          filter = {'tr':{'size':3, 'vec':[1. / 4, 1. / 2, 1. / 4], 'name':'Triangle'},
                    'b3':{'size':5, 'vec':[1. / 16, 1. / 4, 3. / 8, 1. / 4, 1. / 16], 'name':'B3 spline'}}

          if dobdsm: wchain, wopts = self.setpara_bdsm(img)

          n, m = img.ch0_arr.shape

          # Calculate residual image that results from normal (non-wavelet) Gaussian fitting
          Op_make_residimage()(img)
          resid = img.resid_gaus_arr

          lpf = img.opts.atrous_lpf
          if lpf not in ['b3', 'tr']: lpf = 'b3'
          jmax = img.opts.atrous_jmax
          l = len(filter[lpf]['vec'])             # 1st 3 is arbit and 2nd 3 is whats expected for a-trous
          if jmax < 1 or jmax > 15:                   # determine jmax
            # Check if largest island size is
            # smaller than 1/3 of image size. If so, use it to determine jmax.
            min_size = min(resid.shape)
            max_isl_shape = (0, 0)
            for isl in img.islands:
                if isl.image.shape[0] * isl.image.shape[1] > max_isl_shape[0] * max_isl_shape[1]:
                    max_isl_shape = isl.image.shape
            if max_isl_shape != (0, 0) and min(max_isl_shape) < min(resid.shape) / 3.0:
                min_size = min(max_isl_shape) * 4.0
            else:
                min_size = min(resid.shape)
            jmax = int(floor(log((min_size / 3.0 * 3.0 - l) / (l - 1) + 1) / log(2.0) + 1.0)) + 1
            if min_size * 0.55 <= (l + (l - 1) * (2 ** (jmax) - 1)): jmax = jmax - 1
          img.wavelet_lpf = lpf
          img.wavelet_jmax = jmax
          mylog.info("Using " + filter[lpf]['name'] + ' filter with J_max = ' + str(jmax))

          img.atrous_islands = []
          img.atrous_gaussians = []
          img.atrous_sources = []
          img.atrous_opts = []
          img.resid_wavelets_arr = cp(img.resid_gaus_arr)

          im_old = img.resid_wavelets_arr
          total_flux = 0.0
          ntot_wvgaus = 0
          stop_wav = False
          pix_masked = N.where(N.isnan(resid) == True)
          jmin = 1
          if img.opts.ncores is None:
              numcores = 1
          else:
              numcores = img.opts.ncores
          for j in range(jmin, jmax + 1):  # extra +1 is so we can do bdsm on cJ as well
            mylogger.userinfo(mylog, "\nWavelet scale #" + str(j))
            im_new = self.atrous(im_old, filter[lpf]['vec'], lpf, j, numcores=numcores, use_scipy_fft=img.opts.use_scipy_fft)
            im_new[pix_masked] = N.nan  # since fftconvolve wont work with blanked pixels
            if img.opts.atrous_sum:
                w = im_new
            else:
                w = im_old - im_new
            im_old = im_new
            suffix = 'w' + `j`
            filename = img.imagename + '.atrous.' + suffix + '.fits'
            if img.opts.output_all:
                func.write_image_to_file('fits', filename, w, img, bdir)
                mylog.info('%s %s' % ('Wrote ', img.imagename + '.atrous.' + suffix + '.fits'))

            # now do bdsm on each wavelet image.
            if dobdsm:
              wopts['filename'] = filename
              wopts['basedir'] = bdir
              box = img.rms_box[0]
              y1 = (l + (l - 1) * (2 ** (j - 1) - 1))
              bs = max(5 * y1, box)  # changed from 10 to 5
              if bs > min(n, m) / 2:
                wopts['rms_map'] = False
                wopts['mean_map'] = 'const'
                wopts['rms_box'] = None
              else:
                wopts['rms_box'] = (bs, bs/3)
                if hasattr(img, '_adapt_rms_isl_pos'):
                    bs_bright = max(5 * y1, img.rms_box_bright[0])
                    if bs_bright < bs/1.5:
                        wopts['adaptive_rms_box'] = True
                        wopts['rms_box_bright'] = (bs_bright, bs_bright/3)
                    else:
                        wopts['adaptive_rms_box'] = False
              if j <= 3:
                wopts['ini_gausfit'] = 'default'
              else:
                wopts['ini_gausfit'] = 'nobeam'
              wid = (l + (l - 1) * (2 ** (j - 1) - 1))# / 3.0
              b1, b2 = img.pixel_beam()[0:2]
              b1 = b1 * fwsig
              b2 = b2 * fwsig
              cdelt = img.wcs_obj.acdelt[:2]

              wimg = Image(wopts)
              wimg.beam = (sqrt(wid * wid + b1 * b1) * cdelt[0] * 2.0, sqrt(wid * wid + b2 * b2) * cdelt[1] * 2.0, 0.0)
              wimg.orig_beam = img.beam
              wimg.pixel_beam = img.pixel_beam
              wimg.pixel_beamarea = img.pixel_beamarea
              wimg.log = 'Wavelet.'
              wimg.basedir = img.basedir
              wimg.extraparams['bbsprefix'] = suffix
              wimg.extraparams['bbsname'] = img.imagename + '.wavelet'
              wimg.extraparams['bbsappend'] = True
              wimg.bbspatchnum = img.bbspatchnum
              wimg.waveletimage = True
              wimg.j = j
              if hasattr(img, '_adapt_rms_isl_pos'):
                  wimg._adapt_rms_isl_pos = img._adapt_rms_isl_pos


              self.init_image_simple(wimg, img, w, '.atrous.' + suffix)
              for op in wchain:
                op(wimg)
                gc.collect()
                if isinstance(op, Op_islands) and img.opts.atrous_orig_isl:
                    if wimg.nisl > 0:

                        # Find islands that do not share any pixels with
                        # islands in original ch0 image.
                        good_isl = []

                        # Make original rank image boolean; rank counts from 0, with -1 being
                        # outside any island
                        orig_rankim_bool = N.array(img.pyrank + 1, dtype = bool)

                        # Multiply rank images
                        old_islands = orig_rankim_bool * (wimg.pyrank + 1) - 1

                        # Exclude islands that don't overlap with a ch0 island.
                        valid_ids = set(old_islands.flatten())
                        for idx, wvisl in enumerate(wimg.islands):
                            if idx in valid_ids:
                                wvisl.valid = True
                                good_isl.append(wvisl)
                            else:
                                wvisl.valid = False

                        wimg.islands = good_isl
                        wimg.nisl = len(good_isl)
                        mylogger.userinfo(mylog, "Number of islands found", '%i' %
                                  wimg.nisl)

                        # Renumber islands:
                        for wvindx, wvisl in enumerate(wimg.islands):
                            wvisl.island_id = wvindx

                if isinstance(op, Op_gausfit):
                  # If opts.atrous_orig_isl then exclude Gaussians outside of
                  # the original ch0 islands
                  nwvgaus = 0
                  if img.opts.atrous_orig_isl:
                      gaul = wimg.gaussians
                      tot_flux = 0.0

                      if img.ngaus == 0:
                          gaus_id = -1
                      else:
                          gaus_id = img.gaussians[-1].gaus_num
                      wvgaul = []
                      for g in gaul:
                          if not hasattr(g, 'valid'):
                              g.valid = False
                          if not g.valid:
                              try:
                                  isl_id = img.pyrank[int(g.centre_pix[0] + 1), int(g.centre_pix[1] + 1)]
                              except IndexError:
                                  isl_id = -1
                              if isl_id >= 0:
                                  isl = img.islands[isl_id]
                                  gcenter = (g.centre_pix[0] - isl.origin[0],
                                             g.centre_pix[1] - isl.origin[1])
                                  if not isl.mask_active[gcenter]:
                                      gaus_id += 1
                                      gcp = Gaussian(img, g.parameters[:], isl.island_id, gaus_id)
                                      gcp.gaus_num = gaus_id
                                      gcp.wisland_id = g.island_id
                                      gcp.jlevel = j
                                      g.valid = True
                                      isl.gaul.append(gcp)
                                      isl.ngaus += 1
                                      img.gaussians.append(gcp)
                                      nwvgaus += 1
                                      tot_flux += gcp.total_flux
                                  else:
                                      g.valid = False
                                      g.jlevel = 0
                              else:
                                  g.valid = False
                                  g.jlevel = 0
                      vg = []
                      for g in wimg.gaussians:
                          if g.valid:
                              vg.append(g)
                      wimg.gaussians = vg
                      mylogger.userinfo(mylog, "Number of valid wavelet Gaussians", str(nwvgaus))
                  else:
                      # Keep all Gaussians and merge islands that overlap
                      tot_flux = check_islands_for_overlap(img, wimg)

                      # Now renumber the islands and adjust the rank image before going to next wavelet image
                      renumber_islands(img)

              total_flux += tot_flux
              if img.opts.interactive and has_pl:
                  dc = '\033[34;1m'
                  nc = '\033[0m'
                  print dc + '--> Displaying islands and rms image...' + nc
                  if max(wimg.ch0_arr.shape) > 4096:
                      print dc + '--> Image is large. Showing islands only.' + nc
                      wimg.show_fit(rms_image=False, mean_image=False, ch0_image=False,
                        ch0_islands=True, gresid_image=False, sresid_image=False,
                        gmodel_image=False, smodel_image=False, pyramid_srcs=False)
                  else:
                      wimg.show_fit()
                  prompt = dc + "Press enter to continue or 'q' stop fitting wavelet images : " + nc
                  answ = raw_input_no_history(prompt)
                  while answ != '':
                      if answ == 'q':
                          img.wavelet_jmax = j
                          stop_wav = True
                          break
                      answ = raw_input_no_history(prompt)
              if len(wimg.gaussians) > 0:
                img.resid_wavelets_arr = self.subtract_wvgaus(img.opts, img.resid_wavelets_arr, wimg.gaussians, wimg.islands)
                if img.opts.atrous_sum:
                    im_old = self.subtract_wvgaus(img.opts, im_old, wimg.gaussians, wimg.islands)
              if stop_wav == True:
                  break

          pyrank = N.zeros(img.pyrank.shape, dtype=N.int32)
          for i, isl in enumerate(img.islands):
              isl.island_id = i
              for g in isl.gaul:
                  g.island_id = i
              for dg in isl.dgaul:
                  dg.island_id = i
              pyrank[isl.bbox] += N.invert(isl.mask_active) * (i + 1)
          pyrank -= 1 # align pyrank values with island ids and set regions outside of islands to -1
          img.pyrank = pyrank

          pdir = img.basedir + '/misc/'
          img.ngaus += ntot_wvgaus
          img.total_flux_gaus += total_flux
          mylogger.userinfo(mylog, "Total flux density in model on all scales" , '%.3f Jy' % img.total_flux_gaus)
          if img.opts.output_all:
              func.write_image_to_file('fits', img.imagename + '.atrous.cJ.fits',
                                       im_new, img, bdir)
              mylog.info('%s %s' % ('Wrote ', img.imagename + '.atrous.cJ.fits'))
              func.write_image_to_file('fits', img.imagename + '.resid_wavelets.fits',
                                       (img.ch0_arr - img.resid_gaus_arr + img.resid_wavelets_arr), img, bdir + '/residual/')
              mylog.info('%s %s' % ('Wrote ', img.imagename + '.resid_wavelets.fits'))
              func.write_image_to_file('fits', img.imagename + '.model_wavelets.fits',
                                       (img.resid_gaus_arr - img.resid_wavelets_arr), img, bdir + '/model/')
              mylog.info('%s %s' % ('Wrote ', img.imagename + '.model_wavelets.fits'))
          img.completed_Ops.append('wavelet_atrous')
Exemple #15
0
    def __call__(self, img):
        mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Islands")
        opts = img.opts

        minsize = opts.minpix_isl
        if minsize == None:
            minsize = int(img.pixel_beamarea()/3.0) # 1/3 of beam area in pixels
            if minsize < 6:
                minsize = 6 # Need at least 6 pixels to obtain good fits
            mylogger.userinfo(mylog, "Minimum number of pixels per island", '%i' %
                          minsize)
        img.minpix_isl = minsize

        if opts.detection_image != '':
            # Use a different image for island detection. The detection
            # image and the measurement image must have the same shape
            # and be registered. Otherwise, one could reproject the
            # detection image using, e.g., the Kapteyn package.
            #
            # First, set up up an Image object and run a limited
            # op_chain.
            from . import _run_op_list
            mylogger.userinfo(mylog, "\nDetermining islands from detection image")

            det_chain, det_opts = self.setpara_bdsm(img, opts.detection_image)
            det_img = Image(det_opts)
            det_img.log = 'Detection image'
            success = _run_op_list(det_img, det_chain)
            if not success:
                return

            # Check that the ch0 images are the same size
            ch0_map = img.ch0_arr
            det_ch0_map = det_img.ch0_arr
            det_shape = det_ch0_map.shape
            ch0_shape = ch0_map.shape
            if det_shape != ch0_shape:
                raise RuntimeError("Detection image shape does not match that of input image.")

            # Run through islands and correct the image and rms, mean and max values
            img.island_labels = det_img.island_labels
            corr_islands = []
            mean_map = img.mean_arr
            rms_map = img.rms_arr
            for i, isl in enumerate(det_img.islands):
                islcp = isl.copy(img.pixel_beamarea(), image=ch0_map[isl.bbox], mean=mean_map[isl.bbox], rms=rms_map[isl.bbox])
                islcp.island_id = i
                corr_islands.append(islcp)
            img.islands = corr_islands
            img.nisl = len(img.islands)
            img.pyrank = det_img.pyrank
            img.minpix_isl = det_img.minpix_isl
            mylogger.userinfo(mylog, "\nContinuing processing using primary image")
        else:
            if opts.src_ra_dec != None:
                mylogger.userinfo(mylog, "Constructing islands at user-supplied source locations")
                img.islands = self.coords_to_isl(img, opts)
            else:
                img.islands = self.ndimage_alg(img, opts)
            img.nisl = len(img.islands)

            mylogger.userinfo(mylog, "Number of islands found", '%i' %
                              len(img.islands))

            ch0_map = img.ch0_arr
            ch0_shape = ch0_map.shape
            pyrank = N.zeros(ch0_shape, dtype=N.int32) - 1
            for i, isl in enumerate(img.islands):
                isl.island_id = i
                if i == 0:
                    pyrank[isl.bbox] = N.invert(isl.mask_active) - 1
                else:
                    pyrank[isl.bbox] = N.invert(isl.mask_active) * i - isl.mask_active

            if opts.output_all: write_islands(img)
            if opts.savefits_rankim:
                func.write_image_to_file(img.use_io, img.imagename + '_pyrank.fits', pyrank, img)

            img.pyrank = pyrank

        img.completed_Ops.append('islands')
        return img
Exemple #16
0
    def __call__(self, img):
        from time import time
        import functions as func
        import itertools

        mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Gausfit")
        if len(img.islands) == 0:
            img.gaussians = []
            img.ngaus = 0
            img.total_flux_gaus = 0.0
            img.completed_Ops.append('gausfit')
            return img

        bar = statusbar.StatusBar('Fitting islands with Gaussians .......... : ',
                                  0, img.nisl)
        opts = img.opts
        if opts.quiet == False and opts.verbose_fitting == False:
            bar.start()
        iter_ngmax  = 10
        min_maxsize = 50.0
        maxsize = opts.splitisl_maxsize
        min_peak_size = 30.0
        peak_size = opts.peak_maxsize
        if maxsize < min_maxsize:
            maxsize = min_maxsize
            opts.splitisl_maxsize = min_maxsize
        if peak_size < min_peak_size:
            peak_size = min_peak_size
            opts.peak_maxsize = min_peak_size

        # Set up multiproccessing. First create a simple copy of the Image
        # object that contains the minimal data needed.
        opts_dict = opts.to_dict()
        img_simple = Image(opts_dict)
        img_simple.pixel_beamarea = img.pixel_beamarea
        img_simple.pixel_beam = img.pixel_beam
        img_simple.thresh_pix = img.thresh_pix
        img_simple.minpix_isl = img.minpix_isl
        img_simple.clipped_mean = img.clipped_mean
        img_simple.beam2pix = img.beam2pix
        img_simple.beam = img.beam

        # Next, define the weights to use when distributing islands among cores.
        # The weight should scale with the processing time. At the moment
        # we use the island area, but other parameters may be better.
        weights = []
        for isl in img.islands:
            weights.append(isl.size_active)

        # Now call the parallel mapping function. Returns a list of [gaul, fgaul]
        # for each island.
        gaus_list = mp.parallel_map(func.eval_func_tuple,
                    itertools.izip(itertools.repeat(self.process_island),
                    img.islands, itertools.repeat(img_simple),
                    itertools.repeat(opts)), numcores=opts.ncores,
                    bar=bar, weights=weights)

        for isl in img.islands:
            ### now convert gaussians into Gaussian objects and store
            idx = isl.island_id
            gaul = gaus_list[idx][0]
            fgaul = gaus_list[idx][1]
            dgaul = []
            gaul = [Gaussian(img, par, idx, gidx)
                        for (gidx, par) in enumerate(gaul)]

            if len(gaul) == 0:
                # No good Gaussians were fit. In this case, make a dummy
                # Gaussian located at the island center so
                # that the source may still be included in output catalogs.
                # These dummy Gaussians all have an ID of -1. They do not
                # appear in any of the source or island Gaussian lists except
                # the island dgaul list.
                if opts.src_ra_dec != None:
                    # Center the dummy Gaussian on the user-specified source position
                    posn_isl = (isl.shape[0]/2.0, isl.shape[1]/2.0)
                    posn_img = (isl.shape[0]/2.0 + isl.origin[0], isl.shape[1]/2.0 + isl.origin[1])
                    par = [isl.image[posn_isl], posn_img[0], posn_img[1], 0.0, 0.0, 0.0]
                else:
                    # Center the dummy Gaussian on the maximum pixel
                    posn = N.unravel_index(N.argmax(isl.image*~isl.mask_active), isl.shape) + N.array(isl.origin)
                    par = [isl.max_value, posn[0], posn[1], 0.0, 0.0, 0.0]
                dgaul = [Gaussian(img, par, idx, -1)]
                gidx = 0
            fgaul= [Gaussian(img, par, idx, gidx + gidx2 + 1, flag)
                        for (gidx2, (flag, par)) in enumerate(fgaul)]

            isl.gaul = gaul
            isl.fgaul= fgaul
            isl.dgaul = dgaul

        gaussian_list = [g for isl in img.islands for g in isl.gaul]
        img.gaussians = gaussian_list

        ### put in the serial number of the gaussians for the whole image
        n = 0
        nn = 0
        tot_flux = 0.0
        if img.waveletimage:
            # store the wavelet scale for each Gaussian
            # (wavelet img's have a img.j attribute)
            j = img.j
        else:
            j = 0
        for isl in img.islands:
            m = 0
            for g in isl.gaul:
                n += 1; m += 1
                g.gaus_num = n - 1
                tot_flux += g.total_flux
            for dg in isl.dgaul:
                nn -= 1
                dg.gaus_num = nn

            isl.ngaus = m
        img.ngaus = n
        img.total_flux_gaus = tot_flux

        mylogger.userinfo(mylog, "Total number of Gaussians fit to image",
                          str(n))
        if not img._pi and not img.waveletimage:
            mylogger.userinfo(mylog, "Total flux density in model", '%.3f Jy' %
                          tot_flux)

        # Check if model flux is very different from sum of flux in image
        if img.ch0_sum_jy > 0 and not img._pi:
            if img.total_flux_gaus/img.ch0_sum_jy < 0.5 or \
                    img.total_flux_gaus/img.ch0_sum_jy > 2.0:
                mylog.warn('Total flux density in model is %0.2f times sum of pixels '\
                               'in input image. Large residuals may remain.' %
                           (img.total_flux_gaus/img.ch0_sum_jy,))

        # Check if there are many Gaussians with deconvolved size of 0 in one
        # axis but not in the other. Don't bother to do this for wavelet images.
        fraction_1d = self.check_for_1d_gaussians(img)
        if fraction_1d > 0.5 and img.beam != None and img.waveletimage == False:
            mylog.warn('After deconvolution, more than 50% of Gaussians are '\
                           "1-D. Unless you're fitting an extended source, "\
                           "beam may be incorrect.")

        img.completed_Ops.append('gausfit')
        return img
Exemple #17
0
        img, op_chain = get_op_chain(img)
        if op_chain is not None:
            _run_op_list(img, op_chain)
            img._prev_opts = img.opts.to_dict()
        return True
    except RuntimeError, err:
        # Catch and log error
        mylog.error(str(err))

        # Re-throw error if the user is not in the interactive shell
        if img._is_interactive_shell:
            return False
        else:
            raise
    except KeyboardInterrupt:
        mylogger.userinfo(mylog, "\n\033[31;1mAborted\033[0m")
        return False


def get_op_chain(img):
    """Determines the optimal Op chain for an Image object.

    This is useful when reprocessing an Image object. For example,
    if Gaussians were already fit, but the user now wants to use
    shapelets, we do not need to re-run Op_gausfit, etc.

    Note that any new options added to opts.py should also be
    added here. If not, a full reprocessing will be done if the
    new option is changed.
    """
    from . import default_chain
Exemple #18
0
    def __call__(self, img):
        #  for each island, get the gaussians into a list and then send them to process
        #  src_index is source number, starting from 0
        mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Gaul2Srl")
        mylogger.userinfo(mylog, 'Grouping Gaussians into sources')
        img.aperture = img.opts.aperture
        if img.aperture is not None and img.aperture <= 0.0:
            mylog.warn('Specified aperture is <= 0. Skipping aperture fluxes.')
            img.aperture = None

        src_index = -1
        dsrc_index = 0
        sources = []
        dsources = []
        no_gaus_islands = []
        for iisl, isl in enumerate(img.islands):
            isl_sources = []
            isl_dsources = []
            g_list = []
            for g in isl.gaul:
                if g.flag == 0:
                    g_list.append(g)

            if len(g_list) > 0:
                if len(g_list) == 1:
                    src_index, source = self.process_single_gaussian(img,
                                                                     g_list,
                                                                     src_index,
                                                                     code='S')
                    sources.append(source)
                    isl_sources.append(source)
                else:
                    src_index, source = self.process_CM(
                        img, g_list, isl, src_index)
                    sources.extend(source)
                    isl_sources.extend(source)
            else:
                if not img.waveletimage:
                    dg = isl.dgaul[0]
                    no_gaus_islands.append(
                        (isl.island_id, dg.centre_pix[0], dg.centre_pix[1]))
                    # Put in the dummy Source as the source and use negative IDs
                    g_list = isl.dgaul
                    dsrc_index, dsource = self.process_single_gaussian(
                        img, g_list, dsrc_index, code='S')
                    dsources.append(dsource)
                    isl_dsources.append(dsource)

            isl.sources = isl_sources
            isl.dsources = isl_dsources
        img.sources = sources
        img.dsources = dsources
        img.nsrc = src_index + 1
        mylogger.userinfo(mylog, "Number of sources formed from Gaussians",
                          str(img.nsrc))
        if not img.waveletimage and not img._pi and len(
                no_gaus_islands) > 0 and not img.opts.quiet:
            message = 'All Gaussians were flagged for the following island'
            if len(no_gaus_islands) == 1:
                message += ':\n'
            else:
                message += 's:\n'
            for isl_id in no_gaus_islands:
                message += '    Island #%i (x=%i, y=%i)\n' % isl_id
            if len(no_gaus_islands) == 1:
                message += 'Please check this island. If it is a valid island and\n'
            else:
                message += 'Please check these islands. If they are valid islands and\n'
            if img.opts.atrous_do:
                message += 'should be fit, try adjusting the flagging options (use\n'\
                           'show_fit with "ch0_flagged=True" to see the flagged Gaussians).'
            else:
                message += 'should be fit, try adjusting the flagging options (use\n'\
                           'show_fit with "ch0_flagged=True" to see the flagged Gaussians)\n'\
                           'or enabling the wavelet module (with "atrous_do=True").'
            message += '\nTo include empty islands in output source catalogs, set\n'\
                        'incl_empty=True in the write_catalog task.'
            mylog.warning(message)

        img.completed_Ops.append('gaul2srl')
Exemple #19
0
    def __call__(self, img):
        mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Preprocess")
        bstat = func.bstat
        if img.opts.kappa_clip is None:
            kappa = -img.pixel_beamarea()
        else:
            kappa = img.opts.kappa_clip

        if img.opts.polarisation_do:
          pols = ['I', 'Q', 'U', 'V']
          ch0images = [img.ch0_arr, img.ch0_Q_arr, img.ch0_U_arr, img.ch0_V_arr]
          img.clipped_mean_QUV = []
          img.clipped_rms_QUV = []
        else:
          pols = ['I'] # assume I is always present
          ch0images = [img.ch0_arr]

        if hasattr(img, 'rms_mask'):
            mask = img.rms_mask
        else:
            mask = img.mask_arr
        opts = img.opts

        for ipol, pol in enumerate(pols):
          image = ch0images[ipol]

          ### basic stats
          mean, rms, cmean, crms, cnt = bstat(image, mask, kappa)
          if cnt > 198: cmean = mean; crms = rms
          if pol == 'I':
            if func.approx_equal(crms, 0.0, rel=None):
                raise RuntimeError('Clipped rms appears to be zero. Check for regions '\
                                       'with values of 0 and\nblank them (with NaNs) '\
                                       'or use trim_box to exclude them.')
            img.raw_mean    = mean
            img.raw_rms     = rms
            img.clipped_mean= cmean
            img.clipped_rms = crms
            mylog.info('%s %.4f %s %.4f %s ' % ("Raw mean (Stokes I) = ", mean*1000.0, \
                       'mJy and raw rms = ',rms*1000.0, 'mJy'))
            mylog.info('%s %.4f %s %s %.4f %s ' % ("sigma clipped mean (Stokes I) = ", cmean*1000.0, \
                       'mJy and ','sigma clipped rms = ',crms*1000.0, 'mJy'))
          else:
            img.clipped_mean_QUV.append(cmean)
            img.clipped_rms_QUV.append(crms)
            mylog.info('%s %s %s %.4f %s %s %.4f %s ' % ("sigma clipped mean (Stokes ", pol, ") = ", cmean*1000.0, \
                       'mJy and ','sigma clipped rms = ',crms*1000.0, 'mJy'))

        image = img.ch0_arr
        # Check if pixels are outside the universe
        if opts.check_outsideuniv:
            mylogger.userinfo(mylog, "Checking for pixels outside the universe")
            noutside_univ = self.outside_univ(img)
            img.noutside_univ = noutside_univ
            frac_blank = round(float(noutside_univ)/float(image.shape[0]*image.shape[1]),3)
            mylogger.userinfo(mylog, "Number of additional pixels blanked", str(noutside_univ)
                              +' ('+str(frac_blank*100.0)+'%)')
        else:
            noutside_univ = 0

        # If needed, (re)mask the image
        if noutside_univ > 0:
            mask = N.isnan(img.ch0_arr)
            masked = mask.any()
            img.masked = masked
            if masked:
                img.mask_arr = mask
            img.blankpix = N.sum(mask)


        ### max/min pixel value & coordinates
        shape = image.shape[0:2]
        if mask != None:
            img.blankpix = N.sum(mask)
        if img.blankpix == 0:
          max_idx = image.argmax()
          min_idx = image.argmin()
        else:
          max_idx = N.nanargmax(image)
          min_idx = N.nanargmin(image)

        img.maxpix_coord = N.unravel_index(max_idx, shape)
        img.minpix_coord = N.unravel_index(min_idx, shape)
        img.max_value    = image.flat[max_idx]
        img.min_value    = image.flat[min_idx]

        ### Solid angle of the image
        cdelt = N.array(img.wcs_obj.acdelt[:2])
        img.omega = N.product(shape)*abs(N.product(cdelt))/(180.*180./pi/pi)

        ### Total flux in ch0 image
        if 'atrous' in img.filename or img._pi or img.log == 'Detection image':
            # Don't do this estimate for atrous wavelet images
            # or polarized intensity image,
            # as it doesn't give the correct flux. Also, ignore
            # the flux in the detection image, as it's likely
            # wrong (e.g., not corrected for the primary beam).
            img.ch0_sum_jy = 0
        else:
            im_flux = N.nansum(image)/img.pixel_beamarea() # Jy
            img.ch0_sum_jy = im_flux
            mylogger.userinfo(mylog, 'Flux from sum of (non-blank) pixels',
                              '%.3f Jy' % (im_flux,))

        ### if image seems confused, then take background mean as zero instead
        alpha_sourcecounts = 2.5  # approx diff src count slope. 2.2?
        if opts.bmpersrc_th is None:
          n = (image >= 5.*crms).sum()
          if n <= 0:
            n = 1
            mylog.info('No pixels in image > 5-sigma.')
            mylog.info('Taking number of pixels above 5-sigma as 1.')
          img.bmpersrc_th = N.product(shape)/((alpha_sourcecounts-1.)*n)
          mylog.info('%s %6.2f' % ('Estimated bmpersrc_th = ', img.bmpersrc_th))
        else:
          img.bmpersrc_th = opts.bmpersrc_th
          mylog.info('%s %6.2f' % ('Taking default bmpersrc_th = ', img.bmpersrc_th))

        confused = False
        if opts.mean_map == 'default':
          if opts.bmpersrc_th <= 25. or cmean/crms >= 0.1:
            confused = True
        img.confused = confused
        mylog.info('Parameter confused is '+str(img.confused))

        img.completed_Ops.append('preprocess')
        return img
Exemple #20
0
    def __call__(self, img):
        mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Collapse")
        if img.opts.polarisation_do:
            pols = ["I", "Q", "U", "V"]  # make sure I is done first
        else:
            pols = ["I"]  # assume I is always present
            img.ch0_Q_arr = None
            img.ch0_U_arr = None
            img.ch0_V_arr = None

        if img.shape[1] > 1:
            c_mode = img.opts.collapse_mode
            chan0 = img.opts.collapse_ch0
            c_list = img.opts.collapse_av
            c_wts = img.opts.collapse_wt
            if c_list == []:
                c_list = N.arange(img.shape[1])
            if len(c_list) == 1:
                c_mode = "single"
                chan0 = c_list[0]
                img.collapse_ch0 = chan0
            ch0sh = img.image_arr.shape[2:]
            if img.opts.polarisation_do:
                ch0images = ["ch0_arr", "ch0_Q_arr", "ch0_U_arr", "ch0_V_arr"]
            else:
                ch0images = ["ch0_arr"]

            # assume all Stokes images have the same blank pixels as I:
            blank = N.isnan(img.image_arr[0])
            hasblanks = blank.any()
            if img.opts.kappa_clip is None:
                kappa = -img.pixel_beamarea()
            else:
                kappa = img.opts.kappa_clip

            mean, rms, cmean, crms = chan_stats(img, kappa)
            img.channel_mean = mean
            img.channel_rms = rms
            img.channel_clippedmean = cmean
            img.channel_clippedrms = crms

            for ipol, pol in enumerate(pols):
                if c_mode == "single":
                    if pol == "I":
                        ch0 = img.image_arr[0, chan0]
                        img.ch0_arr = ch0
                        mylogger.userinfo(
                            mylog,
                            "Source extraction will be " "done on channel",
                            "%i (%.3f MHz)" % (chan0, img.frequency / 1e6),
                        )
                    else:
                        ch0[:] = img.image_arr[ipol, chan0][:]
                        img.__setattr__(ch0images[ipol][:], ch0)

                if c_mode == "average":
                    if not hasblanks:
                        if pol == "I":
                            ch0, wtarr = avspc_direct(c_list, img.image_arr[0], img.channel_clippedrms, c_wts)
                        else:
                            # use wtarr from the I image, which is always collapsed first
                            ch0, wtarr = avspc_direct(
                                c_list, img.image_arr[ipol], img.channel_clippedrms, c_wts, wtarr=wtarr
                            )
                    else:
                        if pol == "I":
                            ch0, wtarr = avspc_blanks(c_list, img.image_arr[0], img.channel_clippedrms, c_wts)
                        else:
                            # use wtarr from the I image, which is always collapsed first
                            ch0, wtarr = avspc_blanks(
                                c_list, img.image_arr[ipol], img.channel_clippedrms, c_wts, wtarr=wtarr
                            )
                    img.__setattr__(ch0images[ipol][:], ch0)

                    if pol == "I":
                        img.avspc_wtarr = wtarr
                        init_freq_collapse(img, wtarr)
                        if c_wts == "unity":
                            mylogger.userinfo(mylog, "Channels averaged with " "uniform weights")
                        else:
                            mylogger.userinfo(mylog, "Channels averaged with " "weights=(1/rms)^2")
                        mylogger.userinfo(mylog, "Source extraction will be " 'done on averaged ("ch0") image')
                        mylogger.userinfo(mylog, "Frequency of averaged " "image", "%.3f MHz" % (img.frequency / 1e6,))
                        str1 = " ".join(str(n) for n in c_list)
                        mylog.debug("%s %s" % ("Channels averaged : ", str1))
                        str1 = " ".join(["%9.4e" % n for n in wtarr])
                        mylog.debug("%s %s %s" % ("Channel weights : ", str1, '; unity=zero if c_wts="rms"'))

                if img.opts.output_all:
                    func.write_image_to_file(img.use_io, img.imagename + ".ch0_" + pol + ".fits", ch0, img)
                    mylog.debug("%s %s " % ("Writing file ", img.imagename + ".ch0_" + pol + ".fits"))

        else:
            # Only one channel in image
            image = img.image_arr
            img.ch0_arr = image[0, 0]
            mylogger.userinfo(mylog, "Frequency of image", "%.3f MHz" % (img.frequency / 1e6,))
            if img.opts.polarisation_do:
                for pol in pols[1:]:
                    if pol == "Q":
                        img.ch0_Q_arr = image[1, 0][:]
                    if pol == "U":
                        img.ch0_U_arr = image[2, 0][:]
                    if pol == "V":
                        img.ch0_V_arr = image[3, 0][:]

        # create mask if needed (assume all pols have the same mask as I)
        image = img.ch0_arr
        mask = N.isnan(image)
        img.blankpix = N.sum(mask)
        frac_blank = round(float(img.blankpix) / float(image.shape[0] * image.shape[1]), 3)
        mylogger.userinfo(mylog, "Number of blank pixels", str(img.blankpix) + " (" + str(frac_blank * 100.0) + "%)")

        # Check whether the input image might be an AWimage. If so, and there
        # are no blank pixels, tell the user that they might to set blank_limit.
        # Once the AWimager incorporates blanking, this check can be removed.
        if img.opts.blank_limit is None and (
            img.blankpix == 0
            and ("restored" in img.filename.lower() or "corr" in img.filename.lower() or "aw" in img.filename.lower())
        ):
            check_low = True
        else:
            check_low = False

        if img.opts.blank_limit is not None or check_low:
            import scipy
            import sys

            if check_low:
                threshold = 1e-5
            else:
                threshold = img.opts.blank_limit
                mylogger.userinfo(mylog, "Blanking pixels with values " "below %.1e Jy/beam" % (threshold,))
            bad = abs(image) < threshold
            original_stdout = sys.stdout  # keep a reference to STDOUT
            sys.stdout = func.NullDevice()  # redirect the real STDOUT
            count = scipy.signal.convolve2d(bad, N.ones((3, 3)), mode="same")
            sys.stdout = original_stdout  # turn STDOUT back on
            mask_low = count >= 5
            if check_low:
                nlow = len(N.where(mask_low)[0])
                if nlow / float(image.shape[0] * image.shape[1]) > 0.2:
                    mylog.warn(
                        "A significant area of the image has very low values. To blank\nthese regions (e.g., because they are outside the primary beam), set the\nblank_limit option (values of 1e-5 to 1e-4 Jy/beam usually work well).\n"
                    )

            else:
                image[N.where(mask_low)] = N.nan
                mask = N.isnan(image)
                img.blankpix = N.sum(mask)
                frac_blank = round(float(img.blankpix) / float(image.shape[0] * image.shape[1]), 3)
                mylogger.userinfo(
                    mylog, "Total number of blanked pixels", str(img.blankpix) + " (" + str(frac_blank * 100.0) + "%)"
                )

        masked = mask.any()
        img.masked = masked
        if masked:
            img.mask_arr = mask
        else:
            img.mask_arr = None

        if img.blankpix == image.shape[0] * image.shape[1]:
            # ALL pixels are blanked!
            raise RuntimeError("All pixels in the image are blanked.")
        img.completed_Ops.append("collapse")
Exemple #21
0
    def init_beam(self, img):
        """Initialize beam parameters, and conversion routines
        to convert beam to/from pixel coordinates"""
        from const import fwsig
        mylog = mylogger.logging.getLogger("PyBDSM.InitBeam")

        hdr = img.header
        cdelt1, cdelt2 = img.wcs_obj.acdelt[0:2]

        ### define beam conversion routines:
        def beam2pix(x):
            """ Converts beam in deg to pixels. Use when no dependence on
            position is appropriate.

            Input beam angle should be degrees CCW from North at image center.
            The output beam angle is degrees CCW from the +y axis of the image.
            """
            bmaj, bmin, bpa = x
            s1 = abs(bmaj / cdelt1)
            s2 = abs(bmin / cdelt2)
            th = bpa
            return (s1, s2, th)

        def pix2beam(x):
            """ Converts beam in pixels to deg. Use when no dependence on
            position is appropriate.

            Input beam angle should be degrees CCW from the +y axis of the image.
            The output beam angle is degrees CCW from North at image center.
            """
            s1, s2, th = x
            bmaj = abs(s1 * cdelt1)
            bmin = abs(s2 * cdelt2)
            bpa = th
            if bmaj < bmin:
                bmaj, bmin = bmin, bmaj
                bpa += 90.0
            bpa = divmod(bpa, 180)[1] ### bpa lies between 0 and 180
            return [bmaj, bmin, bpa]

        def pixel_beam():
            """Returns the beam in sigma units in pixels"""
            pbeam = beam2pix(img.beam)
            return (pbeam[0]/fwsig, pbeam[1]/fwsig, pbeam[2])

        def pixel_beamarea():
            """Returns the beam area in pixels"""
            pbeam = beam2pix(img.beam)
            return 1.1331 * pbeam[0] * pbeam[1]

        ### Get the beam information from the header
        found = False
        if img.opts.beam is not None:
            beam = img.opts.beam
        else:
            try:
                beam = (hdr['BMAJ'], hdr['BMIN'], hdr['BPA'])
                found = True
            except:
                ### try see if AIPS as put the beam in HISTORY as usual
               for h in hdr.get_history():
                  # Check if h is a string or a FITS Card object (long headers are
                  # split into Cards as of PyFITS 3.0.4)
                  if not isinstance(h, str):
                    hstr = h.value
                  else:
                    hstr = h
                  if N.all(['BMAJ' in hstr, 'BMIN' in hstr, 'BPA' in hstr, 'CLEAN' in hstr]):
                    try:
                        dum, dum, dum, bmaj, dum, bmin, dum, bpa = hstr.split()
                    except ValueError:
                        try:
                            dum, dum, bmaj, dum, bmin, dum, bpa, dum, dum = hstr.split()
                        except ValueError:
                            break
                    beam = (float(bmaj), float(bmin), float(bpa))
                    found = True
            if not found: raise RuntimeError("No beam information found in image header.")

        ### convert beam into pixels (at image center)
        pbeam = beam2pix(beam)
        pbeam = (pbeam[0] / fwsig, pbeam[1] / fwsig, pbeam[2])  # IN SIGMA UNITS

        ### and store it
        img.pix2beam = pix2beam
        img.beam2pix = beam2pix
        img.beam = beam   # FWHM size in degrees
        img.pixel_beam = pixel_beam   # IN SIGMA UNITS in pixels
        img.pixel_beamarea = pixel_beamarea
        mylogger.userinfo(mylog, 'Beam shape (major, minor, pos angle)',
                          '(%.5e, %.5e, %s) degrees' % (beam[0], beam[1],
                                                    round(beam[2], 1)))
Exemple #22
0
    def __call__(self, img):
        mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"RMSimage")
        mylogger.userinfo(mylog, "Calculating background rms and mean images")
        if img.opts.polarisation_do:
            pols = ['I', 'Q', 'U', 'V']
            ch0_images = [img.ch0_arr, img.ch0_Q_arr, img.ch0_U_arr, img.ch0_V_arr]
            cmeans = [img.clipped_mean] + img.clipped_mean_QUV
            crmss = [img.clipped_rms] + img.clipped_rms_QUV
        else:
            pols = ['I'] # assume I is always present
            ch0_images = [img.ch0_arr]
            cmeans = [img.clipped_mean]
            crmss = [img.clipped_rms]

        mask = img.mask_arr
        opts = img.opts
        cdelt = N.array(img.wcs_obj.acdelt[:2])

        # Determine box size for rms/mean map calculations.
        # If user specifies rms_box, use it. Otherwise, use either an
        # adaptive binning scheme that shrinks the box near
        # the brightest sources or estimate rms_box from bright sources.
        #
        # The adaptive scheme calculates the rms/mean map
        # at two different scales:
        #   1) using a large rms_box, set by size of largest source
        #   2) using a small rms_box, set by size of largest bright source
        # Then, the rms and mean values at a given point are determined
        # by a weighted average of the values in the maps at the two
        # scales.
        fwsig = const.fwsig
        min_adapt_threshold = 10.0
        if opts.adaptive_thresh == None:
            adapt_thresh = 50.0
            start_thresh = 500.0
        else:
            adapt_thresh = opts.adaptive_thresh
            if adapt_thresh < min_adapt_threshold:
                adapt_thresh = min_adapt_threshold
                opts.adaptive_thresh = min_adapt_threshold
            start_thresh = adapt_thresh
        brightsize = None
        isl_pos = []
        do_adapt = img.opts.adaptive_rms_box
        img.use_rms_map = None
        img.mean_map_type = None

        # 'size' of brightest source
        kappa1 = 3.0
        try:
            brightsize = int(round(2.*img.beam[0]/cdelt[0]/fwsig*
                               sqrt(2.*log(img.max_value/(kappa1*crms)))))
        except:
            brightsize = int(round(2.*img.beam[0]/cdelt[0]/fwsig))
        mylog.info('Estimated size of brightest source (pixels) = '+str(brightsize))

        # Using clipped mean and rms and a starting threshold of 500 sigma,
        # search for bright sources. If fewer than 5 are found, reduce
        # threshold until limit set by adapt_thresh is hit.
        cmean = cmeans[0]
        crms = crmss[0]
        image = ch0_images[0]
        shape = image.shape
        isl_size_bright = []
        isl_area_highthresh = []
        isl_peak = []
        max_isl_brightsize = 0.0
        threshold = start_thresh
        if do_adapt:
            mylogger.userinfo(mylog, "Using adaptive scaling of rms_box")
            while len(isl_size_bright) < 5 and threshold >= adapt_thresh:
                isl_size_bright=[]
                isl_maxposn = []
                act_pixels = (image-cmean)/threshold >= crms
                threshold *= 0.8
                if isinstance(mask, N.ndarray):
                    act_pixels[mask] = False
                rank = len(image.shape)
                connectivity = nd.generate_binary_structure(rank, rank)
                labels, count = nd.label(act_pixels, connectivity)
                slices = nd.find_objects(labels)
                for idx, s in enumerate(slices):
                    isl_size_bright.append(max([s[0].stop-s[0].start, s[1].stop-s[1].start]))
                    size_area = (labels[s] == idx+1).sum()/img.pixel_beamarea()*2.0
                    isl_area_highthresh.append(size_area)
                    isl_maxposn.append(tuple(N.array(N.unravel_index(N.argmax(image[s]), image[s].shape))+\
                          N.array((s[0].start, s[1].start))))
                    isl_peak.append(nd.maximum(image[s], labels[s], idx+1))

        # Check islands found above at thresh_isl threshold to determine if
        # the bright source is embedded inside a large island or not. If it is,
        # exclude it from the bright-island list. Also find the size of the
        # largest island at this threshold to set the large-scale rms_box
        bright_threshold = threshold
        threshold = 10.0
        act_pixels = (image-cmean)/threshold >= crms
        if isinstance(mask, N.ndarray):
            act_pixels[mask] = False
        rank = len(image.shape)
        connectivity = nd.generate_binary_structure(rank, rank)
        labels, count = nd.label(act_pixels, connectivity)
        slices = nd.find_objects(labels)
        isl_size = []
        isl_size_highthresh = []
        isl_size_lowthresh = []
        isl_snr = []
        thratio = threshold/bright_threshold
        for idx, s in enumerate(slices):
            isl_area_lowthresh = (labels[s] == idx+1).sum()/img.pixel_beamarea()*2.0
            isl_maxposn_lowthresh = tuple(N.array(N.unravel_index(N.argmax(image[s]), image[s].shape))+
                                          N.array((s[0].start, s[1].start)))
            isl_size += [s[0].stop-s[0].start, s[1].stop-s[1].start]
            if do_adapt and isl_maxposn_lowthresh in isl_maxposn:
                bright_indx = isl_maxposn.index(isl_maxposn_lowthresh)
                if isl_area_lowthresh < 25.0 or isl_area_lowthresh/isl_area_highthresh[bright_indx] < 8.0:
                    isl_pos.append(isl_maxposn_lowthresh)
                    isl_size_lowthresh.append(max([s[0].stop-s[0].start, s[1].stop-s[1].start]))
                    isl_size_highthresh.append(isl_size_bright[bright_indx])
                    isl_snr.append(isl_peak[bright_indx]/crms)

        if len(isl_size) == 0:
            max_isl_size = 0.0
        else:
            max_isl_size = max(isl_size)
        mylog.info('Maximum extent of largest 10-sigma island using clipped rms (pixels) = '+str(max_isl_size))
        if len(isl_size_highthresh) == 0:
            max_isl_size_highthresh = 0.0
            max_isl_size_lowthresh = 0.0
        else:
            max_isl_size_highthresh = max(isl_size_highthresh)
            max_isl_size_lowthresh = max(isl_size_lowthresh)
            avg_max_isl_size = (max_isl_size_highthresh + max_isl_size_lowthresh) / 2.0

        if hasattr(img, '_adapt_rms_isl_pos'):
            isl_pos = img._adapt_rms_isl_pos # set isl_pos to existing value (for wavelet analysis)
        if len(isl_pos) == 0:
            # No bright sources found
            do_adapt = False
        else:
            img._adapt_rms_isl_pos = isl_pos
        min_size_allowed = int(img.pixel_beam()[0]*9.0)

        if opts.rms_box is None or (opts.rms_box_bright is None and do_adapt):
            if do_adapt:
                bsize = int(max(brightsize, min_size_allowed, max_isl_size_highthresh*2.0))
            else:
                bsize = int(max(brightsize, min_size_allowed, max_isl_size*2.0))
            bsize2 = int(max(min(image.shape)/10.0, max_isl_size*5.0))
            if bsize < min_size_allowed:
                bsize = min_size_allowed
            if bsize % 10 == 0: bsize += 1
            if bsize2 < min_size_allowed:
                bsize2 = min_size_allowed
            if bsize2 % 10 == 0: bsize2 += 1
            bstep = int(round(min(bsize/3., min(shape)/10.)))
            bstep2 = int(round(min(bsize2/3., min(shape)/10.)))
            if opts.rms_box_bright is None:
                img.rms_box_bright = (bsize, bstep)
            else:
                img.rms_box_bright = opts.rms_box_bright
            if opts.rms_box is None:
                img.rms_box = (bsize2, bstep2)
            else:
                img.rms_box = opts.rms_box
        else:
            if do_adapt:
                img.rms_box_bright = opts.rms_box_bright
                img.rms_box = opts.rms_box
            else:
                img.rms_box_bright = opts.rms_box
                img.rms_box = opts.rms_box

        if opts.kappa_clip is None:
            kappa = -img.pixel_beamarea()
        else:
            kappa = img.opts.kappa_clip

        if do_adapt:
            map_opts = (kappa, img.rms_box_bright, opts.spline_rank)
        else:
            map_opts = (kappa, img.rms_box, opts.spline_rank)

        for ipol, pol in enumerate(pols):
          data = ch0_images[ipol]
          mean = N.zeros(data.shape, dtype=N.float32)
          rms  = N.zeros(data.shape, dtype=N.float32)
          if len(pols) > 1:
              pol_txt = ' (' + pol + ')'
          else:
              pol_txt = ''

          ## calculate rms/mean maps if needed
          if ((opts.rms_map is not False) or (opts.mean_map not in ['zero', 'const'])) and img.rms_box[0] > min(image.shape)/4.0:
            # rms box is too large - just use constant rms and mean
            self.output_rmsbox_size(img)
            mylogger.userinfo(mylog, 'Size of rms_box larger than 1/4 of image size')
            mylogger.userinfo(mylog, 'Using constant background rms and mean')
            img.use_rms_map = False
            img.mean_map_type = 'const'
          else:
            if (opts.rms_map is not False) or (opts.mean_map not in ['zero', 'const']):
              if len(data.shape) == 2:   ## 2d case
                mean, rms = self.calculate_maps(img, data, mean, rms, mask, map_opts, do_adapt=do_adapt,
                                bright_pt_coords=isl_pos, rms_box2=img.rms_box,
                                logname="PyBDSM."+img.log, ncores=img.opts.ncores)
              elif len(data.shape) == 3: ## 3d case
                if not isinstance(mask, N.ndarray):
                  mask = N.zeros(data.shape[0], dtype=bool)
                for i in range(data.shape[0]):
                    ## iterate each plane
                    mean, rms = self.calculate_maps(img, data[i], mean[i], rms[i], mask[i], map_opts,
                                    do_adapt=do_adapt, bright_pt_coords=isl_pos,
                                    rms_box2=img.rms_box, logname="PyBDSM."+img.log,
                                    ncores=img.opts.ncores)
              else:
                mylog.critical('Image shape not handleable' + pol_txt)
                raise RuntimeError("Can't handle array of this shape" + pol_txt)
              self.output_rmsbox_size(img)
              if do_adapt:
                  mylogger.userinfo(mylog, 'Number of sources using small scale', str(len(isl_pos)))
              mylog.info('Background rms and mean images computed' + pol_txt)

            ## check if variation of rms/mean maps is significant enough:
            #       check_rmsmap() sets img.use_rms_map
            #       check_meanmap() sets img.mean_map_type
            if pol == 'I':
                if opts.rms_map is None and img.use_rms_map is None:
                    if do_adapt and len(isl_pos) > 0:
                        # Always use 2d map if there is at least one bright
                        # source and adaptive scaling is desired
                        img.use_rms_map = True
                    else:
                        self.check_rmsmap(img, rms)
                elif opts.rms_map != None:
                    img.use_rms_map = opts.rms_map
                if img.use_rms_map is False:
                    mylogger.userinfo(mylog, 'Using constant background rms')
                else:
                    mylogger.userinfo(mylog, 'Using 2D map for background rms')

                if opts.mean_map == 'default' and img.mean_map_type is None:
                    self.check_meanmap(img, rms)
                elif opts.mean_map != 'default':
                    img.mean_map_type = opts.mean_map
                if img.mean_map_type != 'map':
                    mylogger.userinfo(mylog, 'Using constant background mean')
                else:
                    mylogger.userinfo(mylog, 'Using 2D map for background mean')

          ## if rms map is insignificant, or rms_map==False use const value
          if img.use_rms_map is False:
            if opts.rms_value == None:
              rms[:]  = crmss[ipol]
            else:
              rms[:]  = opts.rms_value
            mylogger.userinfo(mylog, 'Value of background rms' + pol_txt,
                              '%.5f Jy/beam' % rms[0][0])
          else:
            rms_min = N.nanmin(rms)
            rms_max = N.nanmax(rms)
            mylogger.userinfo(mylog, 'Min/max values of background rms map' + pol_txt,
                              '(%.5f, %.5f) Jy/beam' % (rms_min, rms_max))

          if img.mean_map_type != 'map':
            if opts.mean_map == 'zero':
                val = 0.0
            else:
                val = img.clipped_mean
            mean[:] = val
            mylogger.userinfo(mylog, 'Value of background mean' + pol_txt,
                              str(round(val,5))+' Jy/beam')
          else:
            mean_min = N.nanmin(mean)
            mean_max = N.nanmax(mean)
            mylogger.userinfo(mylog, 'Min/max values of background mean map' + pol_txt,
                              '(%.5f, %.5f) Jy/beam' % (mean_min, mean_max))

          if pol == 'I':
            # Apply mask to mean_map and rms_map by setting masked values to NaN
            if isinstance(mask, N.ndarray):
                pix_masked = N.where(mask == True)
                mean[pix_masked] = N.nan
                rms[pix_masked] = N.nan

            img.mean_arr = mean
            img.rms_arr = rms

            if opts.savefits_rmsim or opts.output_all:
              if img.waveletimage:
                  resdir = img.basedir + '/wavelet/background/'
              else:
                  resdir = img.basedir + '/background/'
              if not os.path.exists(resdir): os.makedirs(resdir)
              func.write_image_to_file(img.use_io, img.imagename + '.rmsd_I.fits', rms, img, resdir)
              mylog.info('%s %s' % ('Writing ', resdir+img.imagename+'.rmsd_I.fits'))
            if opts.savefits_meanim or opts.output_all:
              if img.waveletimage:
                  resdir = img.basedir + '/wavelet/background/'
              else:
                  resdir = img.basedir + '/background/'
              if not os.path.exists(resdir): os.makedirs(resdir)
              func.write_image_to_file(img.use_io, img.imagename + '.mean_I.fits', mean, img, resdir)
              mylog.info('%s %s' % ('Writing ', resdir+img.imagename+'.mean_I.fits'))
            if opts.savefits_normim or opts.output_all:
              if img.waveletimage:
                  resdir = img.basedir + '/wavelet/background/'
              else:
                  resdir = img.basedir + '/background/'
              if not os.path.exists(resdir): os.makedirs(resdir)
              zero_pixels = N.where(rms <= 0.0)
              rms_nonzero = rms.copy()
              rms_nonzero[zero_pixels] = N.NaN
              func.write_image_to_file(img.use_io, img.imagename + '.norm_I.fits', (image-mean)/rms_nonzero, img, resdir)
              mylog.info('%s %s' % ('Writing ', resdir+img.imagename+'.norm_I.fits'))
          else:
              img.__setattr__('mean_'+pol+'_arr', mean)
              img.__setattr__('rms_'+pol+'_arr', rms)

        img.completed_Ops.append('rmsimage')
        return img
Exemple #23
0
    def __call__(self, img):
        import time, os
        mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Readimage")

        if img.opts.filename == '':
            raise RuntimeError('Image file name not specified.')

        # Check for trailing "/" in file name (since CASA images are directories).
        # Although the general rule is to not alter the values in opts (only the
        # user should be able to alter these), in this case there is no harm in
        # replacing the file name in opts with the '/' trimmed off.
        if img.opts.filename[-1] == '/':
            img.opts.filename = img.opts.filename[:-1]

        # Determine indir if not explicitly given by user (in img.opts.indir)
        if img.opts.indir is None:
            indir = os.path.dirname(img.opts.filename)
            if indir == '':
                indir = './'
            img.indir = indir
        else:
            img.indir = img.opts.indir

        # Try to trim common extensions from filename and store various
        # paths
        root, ext = os.path.splitext(img.opts.filename)
        if ext in ['.fits', '.FITS', '.image']:
            fname = root
        elif ext in ['.gz', '.GZ']:
            root2, ext2 = os.path.splitext(root)
            if ext2 in ['.fits', '.FITS', '.image']:
                fname = root2
            else:
                fname = root
        else:
            fname = img.opts.filename
        img.filename = img.opts.filename
        img.parentname = fname
        img.imagename = fname + '.pybdsm'
        img.basedir = './' + fname + '_pybdsm/'

        # Read in data and header
        image_file = os.path.basename(img.opts.filename)
        result = read_image_from_file(image_file, img, img.indir)
        if result is None:
            raise RuntimeError("Cannot open file " + repr(image_file) + ". " + img._reason)
        else:
            data, hdr = result

        # Check whether caching is to be used. If it is, set up a
        # temporary directory. The temporary directory will be
        # removed automatically upon exit.
        if img.opts.do_cache:
            img.do_cache = True
        else:
            img.do_cache = False
        if img.do_cache:
            mylog.info('Using disk caching.')
            tmpdir = img.parentname+'_tmp'
            if not os.path.exists(tmpdir):
                os.makedirs(tmpdir)
            img._tempdir_parent = TempDir(tmpdir)
            img.tempdir = TempDir(tempfile.mkdtemp(dir=tmpdir))
            import atexit, shutil
            atexit.register(shutil.rmtree, img._tempdir_parent, ignore_errors=True)
        else:
            img.tempdir = None

        # Store data and header in img. If polarisation_do = False, only store pol == 'I'
        img.nchan = data.shape[1]
        img.nstokes = data.shape[0]
        mylogger.userinfo(mylog, 'Image size',
                          str(data.shape[-2:]) + ' pixels')
        mylogger.userinfo(mylog, 'Number of channels',
                          '%i' % data.shape[1])
        mylogger.userinfo(mylog, 'Number of Stokes parameters',
                          '%i' % data.shape[0])
        if img.opts.polarisation_do and data.shape[0] == 1:
            img.opts.polarisation_do = False
            mylog.warning('Image has Stokes I only. Polarisation module disabled.')

        if img.opts.polarisation_do or data.shape[0] == 1:
            img.image_arr = data
        else:
            img.image_arr = data[0, :].reshape(1, data.shape[1], data.shape[2], data.shape[3])
        img.header = hdr
        img.shape = data.shape
        img.j = 0

        ### initialize wcs conversion routines
        self.init_wcs(img)
        self.init_beam(img)
        self.init_freq(img)
        year, code = self.get_equinox(img)
        if year is None:
            mylog.info('Equinox not found in image header. Assuming J2000.')
            img.equinox = 2000.0
        else:
            mylog.info('Equinox of image is %f.' % year)
            img.equinox = year

        if img.opts.output_all:
            # Set up directory to write output to
            opdir = img.opts.opdir_overwrite
            if opdir not in ['overwrite', 'append']:
                img.opts.opdir_overwrite = 'append'
            if opdir == 'append':
                mylog.info('Appending output files to directory ' + img.basedir)
            else:
                mylog.info('Overwriting output files (if any) in directory ' + img.basedir)
                if os.path.isdir(img.basedir):
                    os.system("rm -fr " + img.basedir + '/*')
            if not os.path.isdir(img.basedir):
                os.makedirs(img.basedir)

            # Now add solname (if any) and time to basedir
            if img.opts.solnname is not None:
                img.basedir += img.opts.solnname + '_'
            img.basedir += time.strftime("%d%b%Y_%H.%M.%S")

            # Make the final output directory
            if not os.path.isdir(img.basedir):
                os.makedirs(img.basedir)

        del data
        img.completed_Ops.append('readimage')
        return img
Exemple #24
0
    def __call__(self, img):

      if img.opts.psf_vary_do:
        mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Psf_Vary")
        mylogger.userinfo(mylog, '\nEstimating PSF variations')
        opts = img.opts
        dir = img.basedir + '/misc/'
        plot = False # debug figures
        image = img.ch0_arr

        try:
            from astropy.io import fits as pyfits
            old_pyfits = False
        except ImportError, err:
            from distutils.version import StrictVersion
            import pyfits
            if StrictVersion(pyfits.__version__) < StrictVersion('2.2'):
                old_pyfits = True
            else:
                old_pyfits = False

        if old_pyfits:
            mylog.warning('PyFITS version is too old: psf_vary module skipped')
            return

        if opts.psf_fwhm is not None:
            # User has specified a constant PSF to use, so skip PSF fitting/etc.
            psf_maj = opts.psf_fwhm[0] # FWHM in deg
            psf_min = opts.psf_fwhm[1] # FWHM in deg
            psf_pa = opts.psf_fwhm[2] # PA in deg
            mylogger.userinfo(mylog, 'Using constant PSF (major, minor, pos angle)',
                  '(%.5e, %.5e, %s) degrees' % (psf_maj, psf_maj,
                                            round(psf_pa, 1)))
        else:
            # Use did not specify a constant PSF to use, so estimate it
            over = 2
            generators = opts.psf_generators; nsig = opts.psf_nsig; kappa2 = opts.psf_kappa2
            snrtop = opts.psf_snrtop; snrbot = opts.psf_snrbot; snrcutstack = opts.psf_snrcutstack
            gencode = opts.psf_gencode; primarygen = opts.psf_primarygen; itess_method = opts.psf_itess_method
            tess_sc = opts.psf_tess_sc; tess_fuzzy= opts.psf_tess_fuzzy
            bright_snr_cut = opts.psf_high_snr
            s_only = opts.psf_stype_only
            if opts.psf_snrcut < 5.0:
                mylogger.userinfo(mylog, "Value of psf_snrcut too low; increasing to 5")
                snrcut = 5.0
            else:
                snrcut = opts.psf_snrcut
            img.psf_snrcut = snrcut
            if opts.psf_high_snr is not None:
                if opts.psf_high_snr < 10.0:
                    mylogger.userinfo(mylog, "Value of psf_high_snr too low; increasing to 10")
                    high_snrcut = 10.0
                else:
                    high_snrcut = opts.psf_high_snr
            else:
                high_snrcut = opts.psf_high_snr
            img.psf_high_snr = high_snrcut

            wtfns=['unity', 'roundness', 'log10', 'sqrtlog10']
            if 0 <= itess_method < 4: tess_method=wtfns[itess_method]
            else: tess_method='unity'

            ### now put all relevant gaussian parameters into a list
            ngaus = img.ngaus
            nsrc = img.nsrc
            num = N.zeros(nsrc, dtype=N.int32)
            peak = N.zeros(nsrc)
            xc = N.zeros(nsrc)
            yc = N.zeros(nsrc)
            bmaj = N.zeros(nsrc)
            bmin = N.zeros(nsrc)
            bpa = N.zeros(nsrc)
            code = N.array(['']*nsrc);
            rms = N.zeros(nsrc)
            src_id_list = []
            for i, src in enumerate(img.sources):
                src_max = 0.0
                for gmax in src.gaussians:
                    # Take only brightest Gaussian per source
                    if gmax.peak_flux > src_max:
                        src_max = gmax.peak_flux
                        g = gmax
                num[i] = i
                peak[i] = g.peak_flux
                xc[i] = g.centre_pix[0]
                yc[i] = g.centre_pix[1]
                bmaj[i] = g.size_pix[0]
                bmin[i] = g.size_pix[1]
                bpa[i] = g.size_pix[2]
                code[i] = img.sources[g.source_id].code
                rms[i] = img.islands[g.island_id].rms
            gauls = (num, peak, xc, yc, bmaj, bmin, bpa, code, rms)
            tr_gauls = self.trans_gaul(gauls)

            # takes gaussians with code=S and snr > snrcut.
            if s_only:
                tr = [n for n in tr_gauls if n[1]/n[8]>snrcut and n[7] == 'S']
            else:
                tr = [n for n in tr_gauls if n[1]/n[8]>snrcut]
            g_gauls = self.trans_gaul(tr)

            # computes statistics of fitted sizes. Same as psfvary_fullstat.f in fBDSM.
            bmaj_a, bmaj_r, bmaj_ca, bmaj_cr, ni = _cbdsm.bstat(bmaj, None, nsig)
            bmin_a, bmin_r, bmin_ca, bmin_cr, ni = _cbdsm.bstat(bmin, None, nsig)
            bpa_a, bpa_r, bpa_ca, bpa_cr, ni = _cbdsm.bstat(bpa, None, nsig)

            # get subset of sources deemed to be unresolved. Same as size_ksclip_wenss.f in fBDSM.
            flag_unresolved = self.get_unresolved(g_gauls, img.beam, nsig, kappa2, over, img.psf_high_snr, plot)
            if len(flag_unresolved) == 0:
                mylog.warning('Insufficient number of sources to determine PSF variation.\nTry changing the PSF options or specify a (constant) PSF with the "psf_fwhm" option')
                return

            # see how much the SNR-weighted sizes of unresolved sources differ from the synthesized beam.
            wtsize_beam_snr = self.av_psf(g_gauls, img.beam, flag_unresolved)

            # filter out resolved sources
            tr_gaul = self.trans_gaul(g_gauls)
            tr = [n for i, n in enumerate(tr_gaul) if flag_unresolved[i]]
            g_gauls = self.trans_gaul(tr)
            mylogger.userinfo(mylog, 'Number of unresolved sources', str(len(g_gauls[0])))

            # get a list of voronoi generators. vorogenS has values (and not None) if generators='field'.
            vorogenP, vorogenS = self.get_voronoi_generators(g_gauls, generators, gencode, snrcut, snrtop, snrbot, snrcutstack)
            mylogger.userinfo(mylog, 'Number of generators for PSF variation', str(len(vorogenP[0])))
            if len(vorogenP[0]) < 3:
                mylog.warning('Insufficient number of generators')
                return

            mylogger.userinfo(mylog, 'Tesselating image')
            # group generators into tiles
            tile_prop = self.edit_vorogenlist(vorogenP, frac=0.9)

            # tesselate the image
            volrank, vorowts = self.tesselate(vorogenP, vorogenS, tile_prop, tess_method, tess_sc, tess_fuzzy, \
                      generators, gencode, image.shape)
            if opts.output_all:
                func.write_image_to_file(img.use_io, img.imagename + '.volrank.fits', volrank, img, dir)

            tile_list, tile_coord, tile_snr = tile_prop
            ntile = len(tile_list)
            bar = statusbar.StatusBar('Determining PSF variation ............... : ', 0, ntile)
            mylogger.userinfo(mylog, 'Number of tiles for PSF variation', str(ntile))

            # For each tile, calculate the weighted averaged psf image. Also for all the sources in the image.
            cdelt = list(img.wcs_obj.acdelt[0:2])
            factor=3.
            psfimages, psfcoords, totpsfimage, psfratio, psfratio_aper = self.psf_in_tile(image, img.beam, g_gauls, \
                       cdelt, factor, snrcutstack, volrank, tile_prop, plot, img)
            npsf = len(psfimages)

        if opts.psf_use_shap:
            if opts.psf_fwhm is None:
                # use totpsfimage to get beta, centre and nmax for shapelet decomposition. Use nmax=5 or 6
                mask=N.zeros(totpsfimage.shape, dtype=bool)
                (m1, m2, m3)=func.moment(totpsfimage, mask)
                betainit=sqrt(m3[0]*m3[1])*2.0  * 1.4
                tshape = totpsfimage.shape
                cen = N.array(N.unravel_index(N.argmax(totpsfimage), tshape))+[1,1]
                cen = tuple(cen)
                nmax = 12
                basis = 'cartesian'
                betarange = [0.5,sqrt(betainit*max(tshape))]
                beta, error  = sh.shape_varybeta(totpsfimage, mask, basis, betainit, cen, nmax, betarange, plot)
                if error == 1: print '  Unable to find minimum in beta'

                # decompose all the psf images using the beta from above
                nmax=12; psf_cf=[]
                for i in range(npsf):
                    psfim = psfimages[i]
                    cf = sh.decompose_shapelets(psfim, mask, basis, beta, cen, nmax, mode='')
                    psf_cf.append(cf)
                    if img.opts.quiet == False:
                        bar.increment()
                bar.stop()

                # transpose the psf image list
                xt, yt = N.transpose(tile_coord)
                tr_psf_cf = N.transpose(N.array(psf_cf))

                # interpolate the coefficients across the image. Ok, interpolate in scipy for
                # irregular grids is crap. doesnt even pass through some of the points.
                # for now, fit polynomial.
                compress = 100.0
                x, y = N.transpose(psfcoords)
                if len(x) < 3:
                    mylog.warning('Insufficient number of tiles to do interpolation of PSF variation')
                    return

                psf_coeff_interp, xgrid, ygrid = self.interp_shapcoefs(nmax, tr_psf_cf, psfcoords, image.shape, \
                         compress, plot)

                psfshape = psfimages[0].shape
                skip = 5
                aa = self.create_psf_grid(psf_coeff_interp, image.shape, xgrid, ygrid, skip, nmax, psfshape, \
                     basis, beta, cen, totpsfimage, plot)
                img.psf_images = aa
        else:
            if opts.psf_fwhm is None:
                if ntile < 4:
                    mylog.warning('Insufficient number of tiles to do interpolation of PSF variation')
                    return
                else:
                    # Fit stacked PSFs with Gaussians and measure aperture fluxes
                    bm_pix = N.array([img.pixel_beam()[0]*fwsig, img.pixel_beam()[1]*fwsig, img.pixel_beam()[2]])
                    psf_maj = N.zeros(npsf)
                    psf_min = N.zeros(npsf)
                    psf_pa = N.zeros(npsf)
                    if img.opts.quiet == False:
                        bar.start()
                    for i in range(ntile):
                        psfim = psfimages[i]
                        mask = N.zeros(psfim.shape, dtype=bool)
                        x_ax, y_ax = N.indices(psfim.shape)
                        maxv = N.max(psfim)
                        p_ini = [maxv, (psfim.shape[0]-1)/2.0*1.1, (psfim.shape[1]-1)/2.0*1.1, bm_pix[0]/fwsig*1.3,
                                 bm_pix[1]/fwsig*1.1, bm_pix[2]*2]
                        para, ierr = func.fit_gaus2d(psfim, p_ini, x_ax, y_ax, mask)
                        ### first extent is major
                        if para[3] < para[4]:
                            para[3:5] = para[4:2:-1]
                            para[5] += 90
                        ### clip position angle
                        para[5] = divmod(para[5], 180)[1]

                        psf_maj[i] = para[3]
                        psf_min[i] = para[4]
                        posang = para[5]
                        while posang >= 180.0:
                            posang -= 180.0
                        psf_pa[i] = posang

                        if img.opts.quiet == False:
                            bar.increment()
                    bar.stop()

                    # Interpolate Gaussian parameters
                    if img.aperture is None:
                        psf_maps = [psf_maj, psf_min, psf_pa, psfratio]
                    else:
                        psf_maps = [psf_maj, psf_min, psf_pa, psfratio, psfratio_aper]
                    nimgs = len(psf_maps)
                    bar = statusbar.StatusBar('Interpolating PSF images ................ : ', 0, nimgs)
                    if img.opts.quiet == False:
                        bar.start()
                    map_list = mp.parallel_map(func.eval_func_tuple,
                        itertools.izip(itertools.repeat(self.interp_prop),
                        psf_maps, itertools.repeat(psfcoords),
                        itertools.repeat(image.shape)), numcores=opts.ncores,
                        bar=bar)
                    if img.aperture is None:
                        psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int = map_list
                    else:
                        psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int, psf_ratio_aper_int = map_list

                    # Smooth if desired
                    if img.opts.psf_smooth is not None:
                        sm_scale = img.opts.psf_smooth / img.pix2beam([1.0, 1.0, 0.0])[0] / 3600.0 # pixels
                        if img.opts.aperture is None:
                            psf_maps = [psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int]
                        else:
                            psf_maps = [psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int, psf_ratio_aper_int]
                        nimgs = len(psf_maps)
                        bar = statusbar.StatusBar('Smoothing PSF images .................... : ', 0, nimgs)
                        if img.opts.quiet == False:
                            bar.start()
                        map_list = mp.parallel_map(func.eval_func_tuple,
                            itertools.izip(itertools.repeat(self.blur_image),
                            psf_maps, itertools.repeat(sm_scale)), numcores=opts.ncores,
                            bar=bar)
                        if img.aperture is None:
                            psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int = map_list
                        else:
                            psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int, psf_ratio_aper_int = map_list

                    # Make sure all smoothed, interpolated images are ndarrays
                    psf_maj_int = N.array(psf_maj_int)
                    psf_min_int = N.array(psf_min_int)
                    psf_pa_int = N.array(psf_pa_int)
                    psf_ratio_int = N.array(psf_ratio_int)
                    if img.aperture is None:
                        psf_ratio_aper_int = N.zeros(psf_maj_int.shape, dtype=N.float32)
                    else:
                        psf_ratio_aper_int = N.array(psf_ratio_aper_int, dtype=N.float32)

                    # Blank with NaNs if needed
                    mask = img.mask_arr
                    if isinstance(mask, N.ndarray):
                        pix_masked = N.where(mask == True)
                        psf_maj_int[pix_masked] = N.nan
                        psf_min_int[pix_masked] = N.nan
                        psf_pa_int[pix_masked] = N.nan
                        psf_ratio_int[pix_masked] = N.nan
                        psf_ratio_aper_int[pix_masked] = N.nan

                    # Store interpolated images. The major and minor axis images are
                    # the sigma in units of arcsec, the PA image in units of degrees east of
                    # north, the ratio images in units of 1/beam.
                    img.psf_vary_maj_arr = psf_maj_int * img.pix2beam([1.0, 1.0, 0.0])[0] * 3600.0 # sigma in arcsec
                    img.psf_vary_min_arr = psf_min_int * img.pix2beam([1.0, 1.0, 0.0])[0] * 3600.0 # sigma in arcsec
                    img.psf_vary_pa_arr = psf_pa_int
                    img.psf_vary_ratio_arr = psf_ratio_int # in 1/beam
                    img.psf_vary_ratio_aper_arr = psf_ratio_aper_int # in 1/beam

                    if opts.output_all:
                        func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_maj.fits', img.psf_vary_maj_arr*fwsig, img, dir)
                        func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_min.fits', img.psf_vary_min_arr*fwsig, img, dir)
                        func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_pa.fits', img.psf_vary_pa_arr, img, dir)
                        func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_ratio.fits', img.psf_vary_ratio_arr, img, dir)
                        func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_ratio_aper.fits', img.psf_vary_ratio_aper_arr, img, dir)

            # Loop through source and Gaussian lists and deconvolve the sizes using appropriate beam
            bar2 = statusbar.StatusBar('Correcting deconvolved source sizes ..... : ', 0, img.nsrc)
            if img.opts.quiet == False:
                bar2.start()
            for src in img.sources:
                src_pos = img.sky2pix(src.posn_sky_centroid)
                src_pos_int = (int(src_pos[0]), int(src_pos[1]))
                gaus_c = img.gaus2pix(src.size_sky, src.posn_sky_centroid)
                if opts.psf_fwhm is None:
                    gaus_bm = [psf_maj_int[src_pos_int]*fwsig, psf_min_int[src_pos_int]*fwsig, psf_pa_int[src_pos_int]]
                else:
                    # Use user-specified constant PSF instead
                    gaus_bm = img.beam2pix(opts.psf_fwhm)
                gaus_dc, err = func.deconv2(gaus_bm, gaus_c)
                src.deconv_size_sky = img.pix2gaus(gaus_dc, src_pos)
                src.deconv_size_skyE = [0.0, 0.0, 0.0]
                for g in src.gaussians:
                    gaus_c = img.gaus2pix(g.size_sky, src.posn_sky_centroid)
                    gaus_dc, err = func.deconv2(gaus_bm, gaus_c)
                    g.deconv_size_sky = img.pix2gaus(gaus_dc, g.centre_pix)
                    g.deconv_size_skyE = [0.0, 0.0, 0.0]
                    if img.opts.quiet == False:
                        bar2.spin()
                if img.opts.quiet == False:
                    bar2.increment()
            bar2.stop()
        img.completed_Ops.append('psf_vary')
Exemple #25
0
    def __call__(self, img):
        mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Threshold ")
        data = img.ch0_arr
        mask = img.mask_arr
        opts = img.opts
        size = N.product(img.ch0_arr.shape)
        sq2  = sqrt(2)

        if img.opts.thresh is None:
            source_p = self.get_srcp(img)
            cutoff = 5.0
            false_p = 0.5*erfc(cutoff/sq2)*size
            if false_p < opts.fdr_ratio*source_p:
                img.thresh = 'hard'
                mylogger.userinfo(mylog, "Expected 5-sigma-clipped false detection rate < fdr_ratio")
                mylogger.userinfo(mylog, "Using sigma-clipping ('hard') thresholding")
            else:
                img.thresh = 'fdr'
                mylogger.userinfo(mylog, "Expected 5-sigma-clipped false detection rate > fdr_ratio")
                mylogger.userinfo(mylog, "Using FDR (False Detection Rate) thresholding")
                mylog.debug('%s %g' % ("Estimated number of source pixels (using sourcecounts.py) is ",source_p))
                mylog.debug('%s %g' % ("Number of false positive pixels expected for 5-sigma is ",false_p))
                mylog.debug("Threshold for pixels set to : "+str.swapcase(img.thresh))
        else:
            img.thresh = img.opts.thresh

        if img.thresh=='fdr':
            cdelt = img.wcs_obj.acdelt[:2]
            bm = (img.beam[0], img.beam[1])
            area_pix = int(round(N.product(bm)/(abs(N.product(cdelt))* \
                                                      pi/(4.0*log(2.0)))))
            s0 = 0
            for i in range(area_pix):
                s0 +=  1.0/(i+1)
            slope = opts.fdr_alpha/s0
            # sort erf of normalised image as vector
            v = N.sort(0.5*erfc(N.ravel((data-img.mean_arr)/img.rms_arr)/sq2))[::-1]
            pcrit = None
            for i,x in enumerate(v):
                if x < slope*i/size:
                    pcrit = x
                    break
            if pcrit is None:
                raise RuntimeError("FDR thresholding failed. Please check the input image for problems.")
            dumr1 = 1.0-2.0*pcrit
            dumr = 8.0/3.0/pi*(pi-3.0)/(4.0-pi)
            # approx for inv(erfc)
            sigcrit = sqrt(-2.0/pi/dumr-log(1.0-dumr1*dumr1)/2.0+  \
                          sqrt((2.0/pi/dumr+log(1.0-dumr1*dumr1)/2.0)* \
                          (2.0/pi/dumr+log(1.0-dumr1*dumr1)/2.0)-      \
                          log(1.0-dumr1*dumr1)/dumr))*sq2
            if pcrit == 0.0:
                img.thresh = 'hard'
            else:
                img.thresh_pix = sigcrit
                mylogger.userinfo(mylog, "FDR threshold (replaces thresh_pix)", str(round(sigcrit, 4)))
        else:
            img.thresh_pix = opts.thresh_pix

        img.completed_Ops.append('threshold')
        return img
Exemple #26
0
    def __call__(self, img):
        mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Collapse")
        if img.opts.polarisation_do:
            pols = ['I', 'Q', 'U', 'V']  # make sure I is done first
        else:
            pols = ['I']  # assume I is always present
            img.ch0_Q_arr = None
            img.ch0_U_arr = None
            img.ch0_V_arr = None

        if img.shape[1] > 1:
            c_mode = img.opts.collapse_mode
            chan0 = img.opts.collapse_ch0
            c_list = img.opts.collapse_av
            c_wts = img.opts.collapse_wt
            if c_list == []: c_list = N.arange(img.shape[1])
            if len(c_list) == 1:
                c_mode = 'single'
                chan0 = c_list[0]
                img.collapse_ch0 = chan0
            ch0sh = img.image_arr.shape[2:]
            if img.opts.polarisation_do:
                ch0images = ['ch0_arr', 'ch0_Q_arr', 'ch0_U_arr', 'ch0_V_arr']
            else:
                ch0images = ['ch0_arr']

            # assume all Stokes images have the same blank pixels as I:
            blank = N.isnan(img.image_arr[0])
            hasblanks = blank.any()
            if img.opts.kappa_clip is None:
                kappa = -img.pixel_beamarea()
            else:
                kappa = img.opts.kappa_clip

            mean, rms, cmean, crms = chan_stats(img, kappa)
            img.channel_mean = mean
            img.channel_rms = rms
            img.channel_clippedmean = cmean
            img.channel_clippedrms = crms

            for ipol, pol in enumerate(pols):
                if c_mode == 'single':
                    if pol == 'I':
                        ch0 = img.image_arr[0, chan0]
                        img.ch0_arr = ch0
                        mylogger.userinfo(mylog, 'Source extraction will be ' \
                                              'done on channel', '%i (%.3f MHz)' % \
                                              (chan0, img.frequency/1e6))
                    else:
                        ch0[:] = img.image_arr[ipol, chan0][:]
                        img.__setattr__(ch0images[ipol][:], ch0)

                if c_mode == 'average':
                    if not hasblanks:
                        if pol == 'I':
                            ch0, wtarr = avspc_direct(c_list, img.image_arr[0],
                                                      img.channel_clippedrms,
                                                      c_wts)
                        else:
                            # use wtarr from the I image, which is always collapsed first
                            ch0, wtarr = avspc_direct(c_list,
                                                      img.image_arr[ipol],
                                                      img.channel_clippedrms,
                                                      c_wts,
                                                      wtarr=wtarr)
                    else:
                        if pol == 'I':
                            ch0, wtarr = avspc_blanks(c_list, img.image_arr[0],
                                                      img.channel_clippedrms,
                                                      c_wts)
                        else:
                            # use wtarr from the I image, which is always collapsed first
                            ch0, wtarr = avspc_blanks(c_list,
                                                      img.image_arr[ipol],
                                                      img.channel_clippedrms,
                                                      c_wts,
                                                      wtarr=wtarr)
                    img.__setattr__(ch0images[ipol][:], ch0)

                    if pol == 'I':
                        img.avspc_wtarr = wtarr
                        init_freq_collapse(img, wtarr)
                        if c_wts == 'unity':
                            mylogger.userinfo(mylog, 'Channels averaged with '\
                                                  'uniform weights')
                        else:
                            mylogger.userinfo(mylog, 'Channels averaged with '\
                                                  'weights=(1/rms)^2')
                        mylogger.userinfo(mylog, 'Source extraction will be '\
                                              'done on averaged ("ch0") image')
                        mylogger.userinfo(mylog, 'Frequency of averaged '\
                                              'image', '%.3f MHz' % \
                                              (img.frequency/1e6,))
                        str1 = " ".join(str(n) for n in c_list)
                        mylog.debug('%s %s' % ('Channels averaged : ', str1))
                        str1 = " ".join(["%9.4e" % n for n in wtarr])
                        mylog.debug('%s %s %s' %
                                    ('Channel weights : ', str1,
                                     '; unity=zero if c_wts="rms"'))

                if img.opts.output_all:
                    func.write_image_to_file(
                        img.use_io, img.imagename + '.ch0_' + pol + '.fits',
                        ch0, img)
                    mylog.debug('%s %s ' % ('Writing file ', img.imagename +
                                            '.ch0_' + pol + '.fits'))

        else:
            # Only one channel in image
            image = img.image_arr
            img.ch0_arr = image[0, 0]
            mylogger.userinfo(mylog, 'Frequency of image',
                              '%.3f MHz' % (img.frequency / 1e6, ))
            if img.opts.polarisation_do:
                for pol in pols[1:]:
                    if pol == 'Q':
                        img.ch0_Q_arr = image[1, 0][:]
                    if pol == 'U':
                        img.ch0_U_arr = image[2, 0][:]
                    if pol == 'V':
                        img.ch0_V_arr = image[3, 0][:]

        # create mask if needed (assume all pols have the same mask as I)
        image = img.ch0_arr
        mask = N.isnan(image)
        img.blankpix = N.sum(mask)
        frac_blank = round(
            float(img.blankpix) / float(image.shape[0] * image.shape[1]), 3)
        mylogger.userinfo(
            mylog, "Number of blank pixels",
            str(img.blankpix) + ' (' + str(frac_blank * 100.0) + '%)')

        # Check whether the input image might be an AWimage. If so, and there
        # are no blank pixels, tell the user that they might to set blank_limit.
        # Once the AWimager incorporates blanking, this check can be removed.
        if img.opts.blank_limit is None and (
                img.blankpix == 0 and
            ('restored' in img.filename.lower() or 'corr'
             in img.filename.lower() or 'aw' in img.filename.lower())):
            check_low = True
        else:
            check_low = False

        if img.opts.blank_limit is not None or check_low:
            import scipy
            import sys
            if check_low:
                threshold = 1e-5
            else:
                threshold = img.opts.blank_limit
                mylogger.userinfo(
                    mylog, "Blanking pixels with values "
                    "below %.1e Jy/beam" % (threshold, ))
            bad = (abs(image) < threshold)
            original_stdout = sys.stdout  # keep a reference to STDOUT
            sys.stdout = func.NullDevice()  # redirect the real STDOUT
            count = scipy.signal.convolve2d(bad, N.ones((3, 3)), mode='same')
            sys.stdout = original_stdout  # turn STDOUT back on
            mask_low = (count >= 5)
            if check_low:
                nlow = len(N.where(mask_low)[0])
                if nlow / float(image.shape[0] * image.shape[1]) > 0.2:
                    mylog.warn(
                        'A significant area of the image has very low values. To blank\nthese regions (e.g., because they are outside the primary beam), set the\nblank_limit option (values of 1e-5 to 1e-4 Jy/beam usually work well).\n'
                    )

            else:
                image[N.where(mask_low)] = N.nan
                mask = N.isnan(image)
                img.blankpix = N.sum(mask)
                frac_blank = round(
                    float(img.blankpix) /
                    float(image.shape[0] * image.shape[1]), 3)
                mylogger.userinfo(
                    mylog, "Total number of blanked pixels",
                    str(img.blankpix) + ' (' + str(frac_blank * 100.0) + '%)')

        masked = mask.any()
        img.masked = masked
        if masked:
            img.mask_arr = mask
        else:
            img.mask_arr = None

        if img.blankpix == image.shape[0] * image.shape[1]:
            # ALL pixels are blanked!
            raise RuntimeError('All pixels in the image are blanked.')
        img.completed_Ops.append('collapse')
Exemple #27
0
 def output_rmsbox_size(self, img):
     """Prints rms/mean box size"""
     opts = img.opts
     do_adapt = opts.adaptive_rms_box
     mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "RMSimage")
     if (opts.rms_map is not False) or (opts.mean_map
                                        not in ['zero', 'const']):
         if do_adapt:
             if opts.rms_box_bright is None:
                 mylogger.userinfo(
                     mylog, 'Derived rms_box (box size, step size)',
                     '(' + str(img.rms_box_bright[0]) + ', ' +
                     str(img.rms_box_bright[1]) + ') pixels (small scale)')
             else:
                 mylogger.userinfo(
                     mylog, 'Using user-specified rms_box',
                     '(' + str(img.rms_box_bright[0]) + ', ' +
                     str(img.rms_box_bright[1]) + ') pixels (small scale)')
             if opts.rms_box is None:
                 mylogger.userinfo(
                     mylog, 'Derived rms_box (box size, step size)',
                     '(' + str(img.rms_box[0]) + ', ' +
                     str(img.rms_box[1]) + ') pixels (large scale)')
             else:
                 mylogger.userinfo(
                     mylog, 'Using user-specified rms_box',
                     '(' + str(img.rms_box[0]) + ', ' +
                     str(img.rms_box[1]) + ') pixels (large scale)')
         else:
             if opts.rms_box is None:
                 mylogger.userinfo(
                     mylog, 'Derived rms_box (box size, step size)',
                     '(' + str(img.rms_box[0]) + ', ' +
                     str(img.rms_box[1]) + ') pixels')
             else:
                 mylogger.userinfo(
                     mylog, 'Using user-specified rms_box',
                     '(' + str(img.rms_box[0]) + ', ' +
                     str(img.rms_box[1]) + ') pixels')
Exemple #28
0
        img, op_chain = get_op_chain(img)
        if op_chain is not None:
            _run_op_list(img, op_chain)
            img._prev_opts = img.opts.to_dict()
        return True
    except RuntimeError, err:
        # Catch and log error
        mylog.error(str(err))

        # Re-throw error if the user is not in the interactive shell
        if img._is_interactive_shell:
            return False
        else:
            raise
    except KeyboardInterrupt:
        mylogger.userinfo(mylog, "\n\033[31;1mAborted\033[0m")
        return False

def get_op_chain(img):
    """Determines the optimal Op chain for an Image object.

    This is useful when reprocessing an Image object. For example,
    if Gaussians were already fit, but the user now wants to use
    shapelets, we do not need to re-run Op_gausfit, etc.

    Note that any new options added to opts.py should also be
    added here. If not, a full reprocessing will be done if the
    new option is changed.
    """
    from . import default_chain
    Op_chain = default_chain[:]
Exemple #29
0
    def __call__(self, img):
        mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "RMSimage")
        mylogger.userinfo(mylog, "Calculating background rms and mean images")
        if img.opts.polarisation_do:
            pols = ['I', 'Q', 'U', 'V']
            ch0_images = [
                img.ch0_arr, img.ch0_Q_arr, img.ch0_U_arr, img.ch0_V_arr
            ]
            cmeans = [img.clipped_mean] + img.clipped_mean_QUV
            crmss = [img.clipped_rms] + img.clipped_rms_QUV
        else:
            pols = ['I']  # assume I is always present
            ch0_images = [img.ch0_arr]
            cmeans = [img.clipped_mean]
            crmss = [img.clipped_rms]

        mask = img.mask_arr
        opts = img.opts
        cdelt = N.array(img.wcs_obj.acdelt[:2])

        # Determine box size for rms/mean map calculations.
        # If user specifies rms_box, use it. Otherwise, use either an
        # adaptive binning scheme that shrinks the box near
        # the brightest sources or estimate rms_box from bright sources.
        #
        # The adaptive scheme calculates the rms/mean map
        # at two different scales:
        #   1) using a large rms_box, set by size of largest source
        #   2) using a small rms_box, set by size of largest bright source
        # Then, the rms and mean values at a given point are determined
        # by a weighted average of the values in the maps at the two
        # scales.
        fwsig = const.fwsig
        min_adapt_threshold = 10.0
        if opts.adaptive_thresh is None:
            adapt_thresh = 50.0
            start_thresh = 500.0
        else:
            adapt_thresh = opts.adaptive_thresh
            if adapt_thresh < min_adapt_threshold:
                adapt_thresh = min_adapt_threshold
                opts.adaptive_thresh = min_adapt_threshold
            start_thresh = adapt_thresh
        brightsize = None
        isl_pos = []
        do_adapt = img.opts.adaptive_rms_box
        img.use_rms_map = None
        img.mean_map_type = None

        # 'size' of brightest source
        kappa1 = 3.0
        try:
            brightsize = int(
                round(2. * img.beam[0] / cdelt[0] / fwsig *
                      sqrt(2. * log(img.max_value / (kappa1 * crms)))))
        except:
            brightsize = int(round(2. * img.beam[0] / cdelt[0] / fwsig))
        mylog.info('Estimated size of brightest source (pixels) = ' +
                   str(brightsize))

        # Using clipped mean and rms and a starting threshold of 500 sigma,
        # search for bright sources. If fewer than 5 are found, reduce
        # threshold until limit set by adapt_thresh is hit.
        cmean = cmeans[0]
        crms = crmss[0]
        image = ch0_images[0]
        shape = image.shape
        isl_size_bright = []
        isl_area_highthresh = []
        isl_peak = []
        max_isl_brightsize = 0.0
        threshold = start_thresh
        if do_adapt:
            mylogger.userinfo(mylog, "Using adaptive scaling of rms_box")
            while len(isl_size_bright) < 5 and threshold >= adapt_thresh:
                isl_size_bright = []
                isl_maxposn = []
                act_pixels = (image - cmean) / threshold >= crms
                threshold *= 0.8
                if isinstance(mask, N.ndarray):
                    act_pixels[mask] = False
                rank = len(image.shape)
                connectivity = nd.generate_binary_structure(rank, rank)
                labels, count = nd.label(act_pixels, connectivity)
                slices = nd.find_objects(labels)
                for idx, s in enumerate(slices):
                    isl_size_bright.append(
                        max([s[0].stop - s[0].start, s[1].stop - s[1].start]))
                    size_area = (labels[s]
                                 == idx + 1).sum() / img.pixel_beamarea() * 2.0
                    isl_area_highthresh.append(size_area)
                    isl_maxposn.append(tuple(N.array(N.unravel_index(N.argmax(image[s]), image[s].shape))+\
                          N.array((s[0].start, s[1].start))))
                    isl_peak.append(nd.maximum(image[s], labels[s], idx + 1))

        # Check islands found above at thresh_isl threshold to determine if
        # the bright source is embedded inside a large island or not. If it is,
        # exclude it from the bright-island list. Also find the size of the
        # largest island at this threshold to set the large-scale rms_box
        bright_threshold = threshold
        threshold = 10.0
        act_pixels = (image - cmean) / threshold >= crms
        if isinstance(mask, N.ndarray):
            act_pixels[mask] = False
        rank = len(image.shape)
        connectivity = nd.generate_binary_structure(rank, rank)
        labels, count = nd.label(act_pixels, connectivity)
        slices = nd.find_objects(labels)
        isl_size = []
        isl_size_highthresh = []
        isl_size_lowthresh = []
        isl_snr = []
        thratio = threshold / bright_threshold
        for idx, s in enumerate(slices):
            isl_area_lowthresh = (labels[s] == idx +
                                  1).sum() / img.pixel_beamarea() * 2.0
            isl_maxposn_lowthresh = tuple(
                N.array(N.unravel_index(N.argmax(image[s]), image[s].shape)) +
                N.array((s[0].start, s[1].start)))
            isl_size += [s[0].stop - s[0].start, s[1].stop - s[1].start]
            if do_adapt and isl_maxposn_lowthresh in isl_maxposn:
                bright_indx = isl_maxposn.index(isl_maxposn_lowthresh)
                if isl_area_lowthresh < 25.0 or isl_area_lowthresh / isl_area_highthresh[
                        bright_indx] < 8.0:
                    isl_pos.append(isl_maxposn_lowthresh)
                    isl_size_lowthresh.append(
                        max([s[0].stop - s[0].start, s[1].stop - s[1].start]))
                    isl_size_highthresh.append(isl_size_bright[bright_indx])
                    isl_snr.append(isl_peak[bright_indx] / crms)

        if len(isl_size) == 0:
            max_isl_size = 0.0
        else:
            max_isl_size = max(isl_size)
        mylog.info(
            'Maximum extent of largest 10-sigma island using clipped rms (pixels) = '
            + str(max_isl_size))
        if len(isl_size_highthresh) == 0:
            max_isl_size_highthresh = 0.0
            max_isl_size_lowthresh = 0.0
        else:
            max_isl_size_highthresh = max(isl_size_highthresh)
            max_isl_size_lowthresh = max(isl_size_lowthresh)
            avg_max_isl_size = (max_isl_size_highthresh +
                                max_isl_size_lowthresh) / 2.0

        if hasattr(img, '_adapt_rms_isl_pos'):
            isl_pos = img._adapt_rms_isl_pos  # set isl_pos to existing value (for wavelet analysis)
        if len(isl_pos) == 0:
            # No bright sources found
            do_adapt = False
        else:
            img._adapt_rms_isl_pos = isl_pos
        min_size_allowed = int(img.pixel_beam()[0] * 9.0)

        if opts.rms_box is None or (opts.rms_box_bright is None and do_adapt):
            if do_adapt:
                bsize = int(
                    max(brightsize, min_size_allowed,
                        max_isl_size_highthresh * 2.0))
            else:
                bsize = int(
                    max(brightsize, min_size_allowed, max_isl_size * 2.0))
            bsize2 = int(max(min(image.shape) / 10.0, max_isl_size * 5.0))
            if bsize < min_size_allowed:
                bsize = min_size_allowed
            if bsize % 10 == 0: bsize += 1
            if bsize2 < min_size_allowed:
                bsize2 = min_size_allowed
            if bsize2 % 10 == 0: bsize2 += 1
            bstep = int(round(min(bsize / 3., min(shape) / 10.)))
            bstep2 = int(round(min(bsize2 / 3., min(shape) / 10.)))
            if opts.rms_box_bright is None:
                img.rms_box_bright = (bsize, bstep)
            else:
                img.rms_box_bright = opts.rms_box_bright
            if opts.rms_box is None:
                img.rms_box = (bsize2, bstep2)
            else:
                img.rms_box = opts.rms_box
        else:
            if do_adapt:
                img.rms_box_bright = opts.rms_box_bright
                img.rms_box = opts.rms_box
            else:
                img.rms_box_bright = opts.rms_box
                img.rms_box = opts.rms_box

        if opts.kappa_clip is None:
            kappa = -img.pixel_beamarea()
        else:
            kappa = img.opts.kappa_clip

        if do_adapt:
            map_opts = (kappa, img.rms_box_bright, opts.spline_rank)
        else:
            map_opts = (kappa, img.rms_box, opts.spline_rank)

        for ipol, pol in enumerate(pols):
            data = ch0_images[ipol]
            mean = N.zeros(data.shape, dtype=N.float32)
            rms = N.zeros(data.shape, dtype=N.float32)
            if len(pols) > 1:
                pol_txt = ' (' + pol + ')'
            else:
                pol_txt = ''

            ## calculate rms/mean maps if needed
            if ((opts.rms_map is not False) or
                (opts.mean_map not in ['zero', 'const'])
                ) and img.rms_box[0] > min(image.shape) / 4.0:
                # rms box is too large - just use constant rms and mean
                self.output_rmsbox_size(img)
                mylogger.userinfo(
                    mylog, 'Size of rms_box larger than 1/4 of image size')
                mylogger.userinfo(mylog,
                                  'Using constant background rms and mean')
                img.use_rms_map = False
                img.mean_map_type = 'const'
            else:
                if (opts.rms_map is not False) or (opts.mean_map
                                                   not in ['zero', 'const']):
                    if len(data.shape) == 2:  ## 2d case
                        mean, rms = self.calculate_maps(
                            img,
                            data,
                            mean,
                            rms,
                            mask,
                            map_opts,
                            do_adapt=do_adapt,
                            bright_pt_coords=isl_pos,
                            rms_box2=img.rms_box,
                            logname="PyBDSM." + img.log,
                            ncores=img.opts.ncores)
                    elif len(data.shape) == 3:  ## 3d case
                        if not isinstance(mask, N.ndarray):
                            mask = N.zeros(data.shape[0], dtype=bool)
                        for i in range(data.shape[0]):
                            ## iterate each plane
                            mean, rms = self.calculate_maps(
                                img,
                                data[i],
                                mean[i],
                                rms[i],
                                mask[i],
                                map_opts,
                                do_adapt=do_adapt,
                                bright_pt_coords=isl_pos,
                                rms_box2=img.rms_box,
                                logname="PyBDSM." + img.log,
                                ncores=img.opts.ncores)
                    else:
                        mylog.critical('Image shape not handleable' + pol_txt)
                        raise RuntimeError("Can't handle array of this shape" +
                                           pol_txt)
                    self.output_rmsbox_size(img)
                    if do_adapt:
                        mylogger.userinfo(
                            mylog, 'Number of sources using small scale',
                            str(len(isl_pos)))
                    mylog.info('Background rms and mean images computed' +
                               pol_txt)

                ## check if variation of rms/mean maps is significant enough:
                #       check_rmsmap() sets img.use_rms_map
                #       check_meanmap() sets img.mean_map_type
                if pol == 'I':
                    if opts.rms_map is None and img.use_rms_map is None:
                        if do_adapt and len(isl_pos) > 0:
                            # Always use 2d map if there is at least one bright
                            # source and adaptive scaling is desired
                            img.use_rms_map = True
                        else:
                            self.check_rmsmap(img, rms)
                    elif opts.rms_map is not None:
                        img.use_rms_map = opts.rms_map
                    if img.use_rms_map is False:
                        mylogger.userinfo(mylog,
                                          'Using constant background rms')
                    else:
                        mylogger.userinfo(mylog,
                                          'Using 2D map for background rms')

                    if opts.mean_map == 'default' and img.mean_map_type is None:
                        self.check_meanmap(img, rms)
                    elif opts.mean_map != 'default':
                        img.mean_map_type = opts.mean_map
                    if img.mean_map_type != 'map':
                        mylogger.userinfo(mylog,
                                          'Using constant background mean')
                    else:
                        mylogger.userinfo(mylog,
                                          'Using 2D map for background mean')

            ## if rms map is insignificant, or rms_map==False use const value
            if img.use_rms_map is False:
                if opts.rms_value is None:
                    rms[:] = crmss[ipol]
                else:
                    rms[:] = opts.rms_value
                mylogger.userinfo(mylog, 'Value of background rms' + pol_txt,
                                  '%.5f Jy/beam' % rms[0][0])
            else:
                rms_min = N.nanmin(rms)
                rms_max = N.nanmax(rms)
                mylogger.userinfo(
                    mylog, 'Min/max values of background rms map' + pol_txt,
                    '(%.5f, %.5f) Jy/beam' % (rms_min, rms_max))

            if img.mean_map_type != 'map':
                if opts.mean_map == 'zero':
                    val = 0.0
                else:
                    val = img.clipped_mean
                mean[:] = val
                mylogger.userinfo(mylog, 'Value of background mean' + pol_txt,
                                  str(round(val, 5)) + ' Jy/beam')
            else:
                mean_min = N.nanmin(mean)
                mean_max = N.nanmax(mean)
                mylogger.userinfo(
                    mylog, 'Min/max values of background mean map' + pol_txt,
                    '(%.5f, %.5f) Jy/beam' % (mean_min, mean_max))

            if pol == 'I':
                # Apply mask to mean_map and rms_map by setting masked values to NaN
                if isinstance(mask, N.ndarray):
                    pix_masked = N.where(mask == True)
                    mean[pix_masked] = N.nan
                    rms[pix_masked] = N.nan

                img.mean_arr = mean
                img.rms_arr = rms

                if opts.savefits_rmsim or opts.output_all:
                    if img.waveletimage:
                        resdir = img.basedir + '/wavelet/background/'
                    else:
                        resdir = img.basedir + '/background/'
                    if not os.path.exists(resdir): os.makedirs(resdir)
                    func.write_image_to_file(img.use_io,
                                             img.imagename + '.rmsd_I.fits',
                                             rms, img, resdir)
                    mylog.info(
                        '%s %s' %
                        ('Writing ', resdir + img.imagename + '.rmsd_I.fits'))
                if opts.savefits_meanim or opts.output_all:
                    if img.waveletimage:
                        resdir = img.basedir + '/wavelet/background/'
                    else:
                        resdir = img.basedir + '/background/'
                    if not os.path.exists(resdir): os.makedirs(resdir)
                    func.write_image_to_file(img.use_io,
                                             img.imagename + '.mean_I.fits',
                                             mean, img, resdir)
                    mylog.info(
                        '%s %s' %
                        ('Writing ', resdir + img.imagename + '.mean_I.fits'))
                if opts.savefits_normim or opts.output_all:
                    if img.waveletimage:
                        resdir = img.basedir + '/wavelet/background/'
                    else:
                        resdir = img.basedir + '/background/'
                    if not os.path.exists(resdir): os.makedirs(resdir)
                    zero_pixels = N.where(rms <= 0.0)
                    rms_nonzero = rms.copy()
                    rms_nonzero[zero_pixels] = N.NaN
                    func.write_image_to_file(img.use_io,
                                             img.imagename + '.norm_I.fits',
                                             (image - mean) / rms_nonzero, img,
                                             resdir)
                    mylog.info(
                        '%s %s' %
                        ('Writing ', resdir + img.imagename + '.norm_I.fits'))
            else:
                img.__setattr__('mean_' + pol + '_arr', mean)
                img.__setattr__('rms_' + pol + '_arr', rms)

        img.completed_Ops.append('rmsimage')
        return img
    def __call__(self, img):

        mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Wavelet")

        if img.opts.atrous_do:
            if img.nisl == 0:
                mylog.warning(
                    "No islands found. Skipping wavelet decomposition.")
                img.completed_Ops.append('wavelet_atrous')
                return

            mylog.info(
                "Decomposing gaussian residual image into a-trous wavelets")
            bdir = img.basedir + '/wavelet/'
            if img.opts.output_all:
                if not os.path.isdir(bdir): os.makedirs(bdir)
                if not os.path.isdir(bdir + '/residual/'):
                    os.makedirs(bdir + '/residual/')
                if not os.path.isdir(bdir + '/model/'):
                    os.makedirs(bdir + '/model/')
            dobdsm = img.opts.atrous_bdsm_do
            filter = {
                'tr': {
                    'size': 3,
                    'vec': [1. / 4, 1. / 2, 1. / 4],
                    'name': 'Triangle'
                },
                'b3': {
                    'size': 5,
                    'vec': [1. / 16, 1. / 4, 3. / 8, 1. / 4, 1. / 16],
                    'name': 'B3 spline'
                }
            }

            if dobdsm: wchain, wopts = self.setpara_bdsm(img)

            n, m = img.ch0_arr.shape

            # Calculate residual image that results from normal (non-wavelet) Gaussian fitting
            Op_make_residimage()(img)
            resid = img.resid_gaus_arr

            lpf = img.opts.atrous_lpf
            if lpf not in ['b3', 'tr']: lpf = 'b3'
            jmax = img.opts.atrous_jmax
            l = len(filter[lpf]['vec']
                    )  # 1st 3 is arbit and 2nd 3 is whats expected for a-trous
            if jmax < 1 or jmax > 15:  # determine jmax
                # Check if largest island size is
                # smaller than 1/3 of image size. If so, use it to determine jmax.
                min_size = min(resid.shape)
                max_isl_shape = (0, 0)
                for isl in img.islands:
                    if isl.image.shape[0] * isl.image.shape[1] > max_isl_shape[
                            0] * max_isl_shape[1]:
                        max_isl_shape = isl.image.shape
                if max_isl_shape != (
                        0, 0) and min(max_isl_shape) < min(resid.shape) / 3.0:
                    min_size = min(max_isl_shape) * 4.0
                else:
                    min_size = min(resid.shape)
                jmax = int(
                    floor(
                        log((min_size / 3.0 * 3.0 - l) /
                            (l - 1) + 1) / log(2.0) + 1.0)) + 1
                if min_size * 0.55 <= (l + (l - 1) * (2**(jmax) - 1)):
                    jmax = jmax - 1
            img.wavelet_lpf = lpf
            img.wavelet_jmax = jmax
            mylog.info("Using " + filter[lpf]['name'] +
                       ' filter with J_max = ' + str(jmax))

            img.atrous_islands = []
            img.atrous_gaussians = []
            img.atrous_sources = []
            img.atrous_opts = []
            img.resid_wavelets_arr = cp(img.resid_gaus_arr)

            im_old = img.resid_wavelets_arr
            total_flux = 0.0
            ntot_wvgaus = 0
            stop_wav = False
            pix_masked = N.where(N.isnan(resid) == True)
            jmin = 1
            if img.opts.ncores is None:
                numcores = 1
            else:
                numcores = img.opts.ncores
            for j in range(jmin, jmax +
                           1):  # extra +1 is so we can do bdsm on cJ as well
                mylogger.userinfo(mylog, "\nWavelet scale #" + str(j))
                im_new = self.atrous(im_old,
                                     filter[lpf]['vec'],
                                     lpf,
                                     j,
                                     numcores=numcores,
                                     use_scipy_fft=img.opts.use_scipy_fft)
                im_new[
                    pix_masked] = N.nan  # since fftconvolve wont work with blanked pixels
                if img.opts.atrous_sum:
                    w = im_new
                else:
                    w = im_old - im_new
                im_old = im_new
                suffix = 'w' + ` j `
                filename = img.imagename + '.atrous.' + suffix + '.fits'
                if img.opts.output_all:
                    func.write_image_to_file('fits', filename, w, img, bdir)
                    mylog.info('%s %s' % ('Wrote ', img.imagename +
                                          '.atrous.' + suffix + '.fits'))

                # now do bdsm on each wavelet image.
                if dobdsm:
                    wopts['filename'] = filename
                    wopts['basedir'] = bdir
                    box = img.rms_box[0]
                    y1 = (l + (l - 1) * (2**(j - 1) - 1))
                    bs = max(5 * y1, box)  # changed from 10 to 5
                    if bs > min(n, m) / 2:
                        wopts['rms_map'] = False
                        wopts['mean_map'] = 'const'
                        wopts['rms_box'] = None
                    else:
                        wopts['rms_box'] = (bs, bs / 3)
                        if hasattr(img, '_adapt_rms_isl_pos'):
                            bs_bright = max(5 * y1, img.rms_box_bright[0])
                            if bs_bright < bs / 1.5:
                                wopts['adaptive_rms_box'] = True
                                wopts['rms_box_bright'] = (bs_bright,
                                                           bs_bright / 3)
                            else:
                                wopts['adaptive_rms_box'] = False
                    if j <= 3:
                        wopts['ini_gausfit'] = 'default'
                    else:
                        wopts['ini_gausfit'] = 'nobeam'
                    wid = (l + (l - 1) * (2**(j - 1) - 1))  # / 3.0
                    b1, b2 = img.pixel_beam()[0:2]
                    b1 = b1 * fwsig
                    b2 = b2 * fwsig
                    cdelt = img.wcs_obj.acdelt[:2]

                    wimg = Image(wopts)
                    wimg.beam = (sqrt(wid * wid + b1 * b1) * cdelt[0] * 2.0,
                                 sqrt(wid * wid + b2 * b2) * cdelt[1] * 2.0,
                                 0.0)
                    wimg.orig_beam = img.beam
                    wimg.pixel_beam = img.pixel_beam
                    wimg.pixel_beamarea = img.pixel_beamarea
                    wimg.log = 'Wavelet.'
                    wimg.basedir = img.basedir
                    wimg.extraparams['bbsprefix'] = suffix
                    wimg.extraparams['bbsname'] = img.imagename + '.wavelet'
                    wimg.extraparams['bbsappend'] = True
                    wimg.bbspatchnum = img.bbspatchnum
                    wimg.waveletimage = True
                    wimg.j = j
                    if hasattr(img, '_adapt_rms_isl_pos'):
                        wimg._adapt_rms_isl_pos = img._adapt_rms_isl_pos

                    self.init_image_simple(wimg, img, w, '.atrous.' + suffix)
                    for op in wchain:
                        op(wimg)
                        gc.collect()
                        if isinstance(op,
                                      Op_islands) and img.opts.atrous_orig_isl:
                            if wimg.nisl > 0:

                                # Find islands that do not share any pixels with
                                # islands in original ch0 image.
                                good_isl = []

                                # Make original rank image boolean; rank counts from 0, with -1 being
                                # outside any island
                                orig_rankim_bool = N.array(img.pyrank + 1,
                                                           dtype=bool)

                                # Multiply rank images
                                old_islands = orig_rankim_bool * (wimg.pyrank +
                                                                  1) - 1

                                # Exclude islands that don't overlap with a ch0 island.
                                valid_ids = set(old_islands.flatten())
                                for idx, wvisl in enumerate(wimg.islands):
                                    if idx in valid_ids:
                                        wvisl.valid = True
                                        good_isl.append(wvisl)
                                    else:
                                        wvisl.valid = False

                                wimg.islands = good_isl
                                wimg.nisl = len(good_isl)
                                mylogger.userinfo(mylog,
                                                  "Number of islands found",
                                                  '%i' % wimg.nisl)

                                # Renumber islands:
                                for wvindx, wvisl in enumerate(wimg.islands):
                                    wvisl.island_id = wvindx

                        if isinstance(op, Op_gausfit):
                            # If opts.atrous_orig_isl then exclude Gaussians outside of
                            # the original ch0 islands
                            nwvgaus = 0
                            if img.opts.atrous_orig_isl:
                                gaul = wimg.gaussians
                                tot_flux = 0.0

                                if img.ngaus == 0:
                                    gaus_id = -1
                                else:
                                    gaus_id = img.gaussians[-1].gaus_num
                                wvgaul = []
                                for g in gaul:
                                    if not hasattr(g, 'valid'):
                                        g.valid = False
                                    if not g.valid:
                                        try:
                                            isl_id = img.pyrank[
                                                int(g.centre_pix[0] + 1),
                                                int(g.centre_pix[1] + 1)]
                                        except IndexError:
                                            isl_id = -1
                                        if isl_id >= 0:
                                            isl = img.islands[isl_id]
                                            gcenter = (g.centre_pix[0] -
                                                       isl.origin[0],
                                                       g.centre_pix[1] -
                                                       isl.origin[1])
                                            if not isl.mask_active[gcenter]:
                                                gaus_id += 1
                                                gcp = Gaussian(
                                                    img, g.parameters[:],
                                                    isl.island_id, gaus_id)
                                                gcp.gaus_num = gaus_id
                                                gcp.wisland_id = g.island_id
                                                gcp.jlevel = j
                                                g.valid = True
                                                isl.gaul.append(gcp)
                                                isl.ngaus += 1
                                                img.gaussians.append(gcp)
                                                nwvgaus += 1
                                                tot_flux += gcp.total_flux
                                            else:
                                                g.valid = False
                                                g.jlevel = 0
                                        else:
                                            g.valid = False
                                            g.jlevel = 0
                                vg = []
                                for g in wimg.gaussians:
                                    if g.valid:
                                        vg.append(g)
                                wimg.gaussians = vg
                                mylogger.userinfo(
                                    mylog, "Number of valid wavelet Gaussians",
                                    str(nwvgaus))
                            else:
                                # Keep all Gaussians and merge islands that overlap
                                tot_flux = check_islands_for_overlap(img, wimg)

                                # Now renumber the islands and adjust the rank image before going to next wavelet image
                                renumber_islands(img)

                    total_flux += tot_flux
                    if img.opts.interactive and has_pl:
                        dc = '\033[34;1m'
                        nc = '\033[0m'
                        print dc + '--> Displaying islands and rms image...' + nc
                        if max(wimg.ch0_arr.shape) > 4096:
                            print dc + '--> Image is large. Showing islands only.' + nc
                            wimg.show_fit(rms_image=False,
                                          mean_image=False,
                                          ch0_image=False,
                                          ch0_islands=True,
                                          gresid_image=False,
                                          sresid_image=False,
                                          gmodel_image=False,
                                          smodel_image=False,
                                          pyramid_srcs=False)
                        else:
                            wimg.show_fit()
                        prompt = dc + "Press enter to continue or 'q' stop fitting wavelet images : " + nc
                        answ = raw_input_no_history(prompt)
                        while answ != '':
                            if answ == 'q':
                                img.wavelet_jmax = j
                                stop_wav = True
                                break
                            answ = raw_input_no_history(prompt)
                    if len(wimg.gaussians) > 0:
                        img.resid_wavelets_arr = self.subtract_wvgaus(
                            img.opts, img.resid_wavelets_arr, wimg.gaussians,
                            wimg.islands)
                        if img.opts.atrous_sum:
                            im_old = self.subtract_wvgaus(
                                img.opts, im_old, wimg.gaussians, wimg.islands)
                    if stop_wav == True:
                        break

            pyrank = N.zeros(img.pyrank.shape, dtype=N.int32)
            for i, isl in enumerate(img.islands):
                isl.island_id = i
                for g in isl.gaul:
                    g.island_id = i
                for dg in isl.dgaul:
                    dg.island_id = i
                pyrank[isl.bbox] += N.invert(isl.mask_active) * (i + 1)
            pyrank -= 1  # align pyrank values with island ids and set regions outside of islands to -1
            img.pyrank = pyrank

            pdir = img.basedir + '/misc/'
            img.ngaus += ntot_wvgaus
            img.total_flux_gaus += total_flux
            mylogger.userinfo(mylog,
                              "Total flux density in model on all scales",
                              '%.3f Jy' % img.total_flux_gaus)
            if img.opts.output_all:
                func.write_image_to_file('fits',
                                         img.imagename + '.atrous.cJ.fits',
                                         im_new, img, bdir)
                mylog.info('%s %s' %
                           ('Wrote ', img.imagename + '.atrous.cJ.fits'))
                func.write_image_to_file(
                    'fits', img.imagename + '.resid_wavelets.fits',
                    (img.ch0_arr - img.resid_gaus_arr +
                     img.resid_wavelets_arr), img, bdir + '/residual/')
                mylog.info('%s %s' %
                           ('Wrote ', img.imagename + '.resid_wavelets.fits'))
                func.write_image_to_file(
                    'fits', img.imagename + '.model_wavelets.fits',
                    (img.resid_gaus_arr - img.resid_wavelets_arr), img,
                    bdir + '/model/')
                mylog.info('%s %s' %
                           ('Wrote ', img.imagename + '.model_wavelets.fits'))
            img.completed_Ops.append('wavelet_atrous')
Exemple #31
0
    def __call__(self, img):

      if img.opts.psf_vary_do:
        mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Psf_Vary")
        mylogger.userinfo(mylog, '\nEstimating PSF variations')
        opts = img.opts
        dir = img.basedir + '/misc/'
        plot = False # debug figures
        image = img.ch0_arr

        try:
            from astropy.io import fits as pyfits
            old_pyfits = False
        except ImportError, err:
            from distutils.version import StrictVersion
            import pyfits
            if StrictVersion(pyfits.__version__) < StrictVersion('2.2'):
                old_pyfits = True
            else:
                old_pyfits = False

        if old_pyfits:
            mylog.warning('PyFITS version is too old: psf_vary module skipped')
            return

        over = 2
        generators = opts.psf_generators; nsig = opts.psf_nsig; kappa2 = opts.psf_kappa2
        snrtop = opts.psf_snrtop; snrbot = opts.psf_snrbot; snrcutstack = opts.psf_snrcutstack
        gencode = opts.psf_gencode; primarygen = opts.psf_primarygen; itess_method = opts.psf_itess_method
        tess_sc = opts.psf_tess_sc; tess_fuzzy= opts.psf_tess_fuzzy
        bright_snr_cut = opts.psf_high_snr
        s_only = opts.psf_stype_only
        if opts.psf_snrcut < 5.0:
            mylogger.userinfo(mylog, "Value of psf_snrcut too low; increasing to 5")
            snrcut = 5.0
        else:
            snrcut = opts.psf_snrcut
        img.psf_snrcut = snrcut
        if opts.psf_high_snr != None:
            if opts.psf_high_snr < 10.0:
                mylogger.userinfo(mylog, "Value of psf_high_snr too low; increasing to 10")
                high_snrcut = 10.0
            else:
                high_snrcut = opts.psf_high_snr
        else:
            high_snrcut = opts.psf_high_snr
        img.psf_high_snr = high_snrcut

        wtfns=['unity', 'roundness', 'log10', 'sqrtlog10']
        if 0 <= itess_method < 4: tess_method=wtfns[itess_method]
        else: tess_method='unity'

        ### now put all relevant gaussian parameters into a list
        ngaus = img.ngaus
        nsrc = img.nsrc
        num = N.zeros(nsrc, dtype=N.int32)
        peak = N.zeros(nsrc)
        xc = N.zeros(nsrc)
        yc = N.zeros(nsrc)
        bmaj = N.zeros(nsrc)
        bmin = N.zeros(nsrc)
        bpa = N.zeros(nsrc)
        code = N.array(['']*nsrc);
        rms = N.zeros(nsrc)
        src_id_list = []
        for i, src in enumerate(img.sources):
            src_max = 0.0
            for gmax in src.gaussians:
                # Take only brightest Gaussian per source
                if gmax.peak_flux > src_max:
                    src_max = gmax.peak_flux
                    g = gmax
            num[i] = i
            peak[i] = g.peak_flux
            xc[i] = g.centre_pix[0]
            yc[i] = g.centre_pix[1]
            bmaj[i] = g.size_pix[0]
            bmin[i] = g.size_pix[1]
            bpa[i] = g.size_pix[2]
            code[i] = img.sources[g.source_id].code
            rms[i] = img.islands[g.island_id].rms
        gauls = (num, peak, xc, yc, bmaj, bmin, bpa, code, rms)
        tr_gauls = self.trans_gaul(gauls)

        # takes gaussians with code=S and snr > snrcut.
        if s_only:
            tr = [n for n in tr_gauls if n[1]/n[8]>snrcut and n[7] == 'S']
        else:
            tr = [n for n in tr_gauls if n[1]/n[8]>snrcut]
        g_gauls = self.trans_gaul(tr)

        # computes statistics of fitted sizes. Same as psfvary_fullstat.f in fBDSM.
        bmaj_a, bmaj_r, bmaj_ca, bmaj_cr, ni = _cbdsm.bstat(bmaj, None, nsig)
        bmin_a, bmin_r, bmin_ca, bmin_cr, ni = _cbdsm.bstat(bmin, None, nsig)
        bpa_a, bpa_r, bpa_ca, bpa_cr, ni = _cbdsm.bstat(bpa, None, nsig)

        # get subset of sources deemed to be unresolved. Same as size_ksclip_wenss.f in fBDSM.
        flag_unresolved = self.get_unresolved(g_gauls, img.beam, nsig, kappa2, over, img.psf_high_snr, plot)

        # see how much the SNR-weighted sizes of unresolved sources differ from the synthesized beam.
        wtsize_beam_snr = self.av_psf(g_gauls, img.beam, flag_unresolved)

        # filter out resolved sources
        tr_gaul = self.trans_gaul(g_gauls)
        tr = [n for i, n in enumerate(tr_gaul) if flag_unresolved[i]]
        g_gauls = self.trans_gaul(tr)
        mylogger.userinfo(mylog, 'Number of unresolved sources', str(len(g_gauls[0])))

        # get a list of voronoi generators. vorogenS has values (and not None) if generators='field'.
        vorogenP, vorogenS = self.get_voronoi_generators(g_gauls, generators, gencode, snrcut, snrtop, snrbot, snrcutstack)
        mylogger.userinfo(mylog, 'Number of generators for PSF variation', str(len(vorogenP[0])))
        if len(vorogenP[0]) < 3:
            mylog.warning('Insufficient number of generators')
            return

        mylogger.userinfo(mylog, 'Tesselating image')
        # group generators into tiles
        tile_prop = self.edit_vorogenlist(vorogenP, frac=0.9)

        # tesselate the image
        volrank, vorowts = self.tesselate(vorogenP, vorogenS, tile_prop, tess_method, tess_sc, tess_fuzzy, \
                  generators, gencode, image.shape)
        if opts.output_all:
            func.write_image_to_file(img.use_io, img.imagename + '.volrank.fits', volrank, img, dir)

        tile_list, tile_coord, tile_snr = tile_prop
        ntile = len(tile_list)
        bar = statusbar.StatusBar('Determining PSF variation ............... : ', 0, ntile)
        mylogger.userinfo(mylog, 'Number of tiles for PSF variation', str(ntile))

        # For each tile, calculate the weighted averaged psf image. Also for all the sources in the image.
        cdelt = list(img.wcs_obj.acdelt[0:2])
        factor=3.
        psfimages, psfcoords, totpsfimage, psfratio, psfratio_aper = self.psf_in_tile(image, img.beam, g_gauls, \
                   cdelt, factor, snrcutstack, volrank, tile_prop, plot, img)
        npsf = len(psfimages)

        if opts.psf_use_shap:
            # use totpsfimage to get beta, centre and nmax for shapelet decomposition. Use nmax=5 or 6
            mask=N.zeros(totpsfimage.shape, dtype=bool)
            (m1, m2, m3)=func.moment(totpsfimage, mask)
            betainit=sqrt(m3[0]*m3[1])*2.0  * 1.4
            tshape = totpsfimage.shape
            cen = N.array(N.unravel_index(N.argmax(totpsfimage), tshape))+[1,1]
            cen = tuple(cen)
            nmax = 12
            basis = 'cartesian'
            betarange = [0.5,sqrt(betainit*max(tshape))]
            beta, error  = sh.shape_varybeta(totpsfimage, mask, basis, betainit, cen, nmax, betarange, plot)
            if error == 1: print '  Unable to find minimum in beta'

            # decompose all the psf images using the beta from above
            nmax=12; psf_cf=[]
            for i in range(npsf):
                psfim = psfimages[i]
                cf = sh.decompose_shapelets(psfim, mask, basis, beta, cen, nmax, mode='')
                psf_cf.append(cf)
                if img.opts.quiet == False:
                    bar.increment()
            bar.stop()

            # transpose the psf image list
            xt, yt = N.transpose(tile_coord)
            tr_psf_cf = N.transpose(N.array(psf_cf))

            # interpolate the coefficients across the image. Ok, interpolate in scipy for
            # irregular grids is crap. doesnt even pass through some of the points.
            # for now, fit polynomial.
            compress = 100.0
            x, y = N.transpose(psfcoords)
            if len(x) < 3:
                mylog.warning('Insufficient number of tiles to do interpolation of PSF variation')
                return

            psf_coeff_interp, xgrid, ygrid = self.interp_shapcoefs(nmax, tr_psf_cf, psfcoords, image.shape, \
                     compress, plot)

            psfshape = psfimages[0].shape
            skip = 5
            aa = self.create_psf_grid(psf_coeff_interp, image.shape, xgrid, ygrid, skip, nmax, psfshape, \
                 basis, beta, cen, totpsfimage, plot)
            img.psf_images = aa
        else:
            if ntile < 4:
                mylog.warning('Insufficient number of tiles to do interpolation of PSF variation')
                return
            else:
                # Fit stacked PSFs with Gaussians and measure aperture fluxes
                bm_pix = N.array([img.pixel_beam()[0]*fwsig, img.pixel_beam()[1]*fwsig, img.pixel_beam()[2]])
                psf_maj = N.zeros(npsf)
                psf_min = N.zeros(npsf)
                psf_pa = N.zeros(npsf)
                if img.opts.quiet == False:
                    bar.start()
                for i in range(ntile):
                    psfim = psfimages[i]
                    mask = N.zeros(psfim.shape, dtype=bool)
                    x_ax, y_ax = N.indices(psfim.shape)
                    maxv = N.max(psfim)
                    p_ini = [maxv, (psfim.shape[0]-1)/2.0*1.1, (psfim.shape[1]-1)/2.0*1.1, bm_pix[0]/fwsig*1.3,
                             bm_pix[1]/fwsig*1.1, bm_pix[2]*2]
                    para, ierr = func.fit_gaus2d(psfim, p_ini, x_ax, y_ax, mask)
                    ### first extent is major
                    if para[3] < para[4]:
                        para[3:5] = para[4:2:-1]
                        para[5] += 90
                    ### clip position angle
                    para[5] = divmod(para[5], 180)[1]

                    psf_maj[i] = para[3]
                    psf_min[i] = para[4]
                    posang = para[5]
                    while posang >= 180.0:
                        posang -= 180.0
                    psf_pa[i] = posang

                    if img.opts.quiet == False:
                        bar.increment()
                bar.stop()

                # Interpolate Gaussian parameters
                if img.aperture == None:
                    psf_maps = [psf_maj, psf_min, psf_pa, psfratio]
                else:
                    psf_maps = [psf_maj, psf_min, psf_pa, psfratio, psfratio_aper]
                nimgs = len(psf_maps)
                bar = statusbar.StatusBar('Interpolating PSF images ................ : ', 0, nimgs)
                if img.opts.quiet == False:
                    bar.start()
                map_list = mp.parallel_map(func.eval_func_tuple,
                    itertools.izip(itertools.repeat(self.interp_prop),
                    psf_maps, itertools.repeat(psfcoords),
                    itertools.repeat(image.shape)), numcores=opts.ncores,
                    bar=bar)
                if img.aperture == None:
                    psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int = map_list
                else:
                    psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int, psf_ratio_aper_int = map_list

                # Smooth if desired
                if img.opts.psf_smooth != None:
                    sm_scale = img.opts.psf_smooth / img.pix2beam([1.0, 1.0, 0.0])[0] / 3600.0 # pixels
                    if img.opts.aperture == None:
                        psf_maps = [psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int]
                    else:
                        psf_maps = [psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int, psf_ratio_aper_int]
                    nimgs = len(psf_maps)
                    bar = statusbar.StatusBar('Smoothing PSF images .................... : ', 0, nimgs)
                    if img.opts.quiet == False:
                        bar.start()
                    map_list = mp.parallel_map(func.eval_func_tuple,
                        itertools.izip(itertools.repeat(self.blur_image),
                        psf_maps, itertools.repeat(sm_scale)), numcores=opts.ncores,
                        bar=bar)
                    if img.aperture == None:
                        psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int = map_list
                    else:
                        psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int, psf_ratio_aper_int = map_list

                # Make sure all smoothed, interpolated images are ndarrays
                psf_maj_int = N.array(psf_maj_int)
                psf_min_int = N.array(psf_min_int)
                psf_pa_int = N.array(psf_pa_int)
                psf_ratio_int = N.array(psf_ratio_int)
                if img.aperture == None:
                    psf_ratio_aper_int = N.zeros(psf_maj_int.shape, dtype=N.float32)
                else:
                    psf_ratio_aper_int = N.array(psf_ratio_aper_int, dtype=N.float32)

                # Blank with NaNs if needed
                mask = img.mask_arr
                if isinstance(mask, N.ndarray):
                    pix_masked = N.where(mask == True)
                    psf_maj_int[pix_masked] = N.nan
                    psf_min_int[pix_masked] = N.nan
                    psf_pa_int[pix_masked] = N.nan
                    psf_ratio_int[pix_masked] = N.nan
                    psf_ratio_aper_int[pix_masked] = N.nan

                # Store interpolated images. The major and minor axis images are
                # the sigma in units of arcsec, the PA image in units of degrees east of
                # north, the ratio images in units of 1/beam.
                img.psf_vary_maj_arr = psf_maj_int * img.pix2beam([1.0, 1.0, 0.0])[0] * 3600.0 # sigma in arcsec
                img.psf_vary_min_arr = psf_min_int * img.pix2beam([1.0, 1.0, 0.0])[0] * 3600.0 # sigma in arcsec
                img.psf_vary_pa_arr = psf_pa_int
                img.psf_vary_ratio_arr = psf_ratio_int # in 1/beam
                img.psf_vary_ratio_aper_arr = psf_ratio_aper_int # in 1/beam

                if opts.output_all:
                    func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_maj.fits', img.psf_vary_maj_arr*fwsig, img, dir)
                    func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_min.fits', img.psf_vary_min_arr*fwsig, img, dir)
                    func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_pa.fits', img.psf_vary_pa_arr, img, dir)
                    func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_ratio.fits', img.psf_vary_ratio_arr, img, dir)
                    func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_ratio_aper.fits', img.psf_vary_ratio_aper_arr, img, dir)

                # Loop through source and Gaussian lists and deconvolve the sizes using appropriate beam
                bar2 = statusbar.StatusBar('Correcting deconvolved source sizes ..... : ', 0, img.nsrc)
                if img.opts.quiet == False:
                    bar2.start()
                for src in img.sources:
                    src_pos = img.sky2pix(src.posn_sky_centroid)
                    src_pos_int = (int(src_pos[0]), int(src_pos[1]))
                    gaus_c = img.gaus2pix(src.size_sky, src.posn_sky_centroid)
                    gaus_bm = [psf_maj_int[src_pos_int]*fwsig, psf_min_int[src_pos_int]*fwsig, psf_pa_int[src_pos_int]*fwsig]
                    gaus_dc, err = func.deconv2(gaus_bm, gaus_c)
                    src.deconv_size_sky = img.pix2gaus(gaus_dc, src_pos)
                    src.deconv_size_skyE = [0.0, 0.0, 0.0]
                    for g in src.gaussians:
                        gaus_c = img.gaus2pix(g.size_sky, src.posn_sky_centroid)
                        gaus_dc, err = func.deconv2(gaus_bm, gaus_c)
                        g.deconv_size_sky = img.pix2gaus(gaus_dc, g.centre_pix)
                        g.deconv_size_skyE = [0.0, 0.0, 0.0]
                        if img.opts.quiet == False:
                            bar2.spin()
                    if img.opts.quiet == False:
                        bar2.increment()
                bar2.stop()
        img.completed_Ops.append('psf_vary')
Exemple #32
0
    def __call__(self, img):
        mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Polarisatn")
        if img.opts.polarisation_do:
            mylog.info('Extracting polarisation properties for all sources')
            pols = ['I', 'Q', 'U', 'V']

            # Run gausfit and gual2srl on PI image to look for polarized sources
            # undetected in I
            fit_PI = img.opts.pi_fit
            n_new = 0
            ch0_pi = N.sqrt(img.ch0_Q_arr**2 + img.ch0_U_arr**2)
            img.ch0_pi_arr = ch0_pi

            if fit_PI:
                from . import _run_op_list
                mylogger.userinfo(mylog, "\nChecking PI image for new sources")

                mask = img.mask_arr
                minsize = img.opts.minpix_isl

                # Set up image object for PI image.
                pi_chain, pi_opts = self.setpara_bdsm(img)
                pimg = Image(pi_opts)
                pimg.beam = img.beam
                pimg.pixel_beam = img.pixel_beam
                pimg.pixel_beamarea = img.pixel_beamarea
                pimg.log = 'PI.'
                pimg.pix2beam = img.pix2beam
                pimg.beam2pix = img.beam2pix
                pimg.pix2gaus = img.pix2gaus
                pimg.gaus2pix = img.gaus2pix
                pimg.pix2sky = img.pix2sky
                pimg.sky2pix = img.sky2pix
                pimg.pix2coord = img.pix2coord
                pimg.wcs_obj = img.wcs_obj
                pimg.mask_arr = mask
                pimg.masked = img.masked
                pimg.ch0_arr = ch0_pi
                pimg._pi = True

                success = _run_op_list(pimg, pi_chain)
                if not success:
                    return

                img.pi_islands = pimg.islands
                img.pi_gaussians = pimg.gaussians
                img.pi_sources = pimg.sources

                # Now check for new sources in the PI image that are not
                # found in the Stokes I image. If any new sources are found,
                # adjust their IDs to follow after those found in I.
                new_isl = []
                new_src = []
                new_gaus = []
                n_new_src = 0
                isl_id = img.islands[-1].island_id
                src_id = img.sources[-1].source_id
                gaus_id = img.gaussians[-1].gaus_num
                for pi_isl in pimg.islands:
                    new_sources = []
                    for pi_src in pi_isl.sources:
                        if img.pyrank[int(img.sky2pix(pi_src.posn_sky_max)[0]),
                                      int(img.sky2pix(pi_src.posn_sky_max)[1]
                                          )] == -1:
                            src_id += 1
                            pi_src._pi = True
                            pi_src.island_id = isl_id
                            pi_src.source_id = src_id
                            pi_src.spec_indx = N.NaN
                            pi_src.e_spec_indx = N.NaN
                            pi_src.spec_norm = N.NaN
                            pi_src.specin_flux = [N.NaN]
                            pi_src.specin_fluxE = [N.NaN]
                            pi_src.specin_freq = [N.NaN]
                            pi_src.specin_freq0 = N.NaN
                            new_sources.append(pi_src)
                            new_src.append(pi_src)
                            n_new_src += 1
                            for g in pi_src.gaussians:
                                gaus_id += 1
                                new_gaus.append(g)
                                g.gaus_num = gaus_id
                    if len(new_sources) > 0:
                        isl_id += 1
                        pi_isl.sources = new_sources
                        pi_isl.island_id = isl_id
                        pi_isl._pi = True
                        new_isl.append(pi_isl)

                n_new = len(new_isl)
                mylogger.userinfo(mylog, "New sources found in PI image",
                                  '%i (%i total)' % (n_new, img.nsrc + n_new))

            if n_new > 0:
                img.islands += new_isl
                img.sources += new_src
                img.gaussians += new_gaus
                img.nsrc += n_new_src

            bar = statusbar.StatusBar(
                'Calculating polarisation properties ....  : ', 0, img.nsrc)
            if img.opts.quiet == False:
                bar.start()

            for isl in img.islands:
                isl_bbox = isl.bbox
                ch0_I = img.ch0_arr[isl_bbox]
                ch0_Q = img.ch0_Q_arr[isl_bbox]
                ch0_U = img.ch0_U_arr[isl_bbox]
                ch0_V = img.ch0_V_arr[isl_bbox]
                ch0_images = [ch0_I, ch0_Q, ch0_U, ch0_V]

                for i, src in enumerate(isl.sources):
                    # For each source, assume the morphology does not change
                    # across the Stokes cube. This assumption allows us to fit
                    # the Gaussians of each source to each Stokes image by
                    # simply fitting only the overall normalizations of the
                    # individual Gaussians.
                    #
                    # First, fit all source Gaussians to each Stokes image:
                    x, y = N.mgrid[isl_bbox]
                    gg = src.gaussians
                    fitfix = N.ones(len(gg))  # fit only normalization
                    srcmask = isl.mask_active
                    total_flux = N.zeros(
                        (4, len(fitfix)), dtype=N.float32
                    )  # array of fluxes: N_Stokes x N_Gaussians
                    errors = N.zeros(
                        (4, len(fitfix)), dtype=N.float32
                    )  # array of fluxes: N_Stokes x N_Gaussians

                    for sind, image in enumerate(ch0_images):
                        if (sind == 0 and hasattr(src, '_pi')
                            ) or sind > 0:  # Fit I only for PI sources
                            p, ep = func.fit_mulgaus2d(image, gg, x, y,
                                                       srcmask, fitfix)
                            for ig in range(len(fitfix)):
                                center_pix = (p[ig * 6 + 1], p[ig * 6 + 2])
                                bm_pix = N.array([
                                    img.pixel_beam()[0],
                                    img.pixel_beam()[1],
                                    img.pixel_beam()[2]
                                ])
                                total_flux[sind, ig] = p[ig * 6] * p[
                                    ig * 6 + 3] * p[ig * 6 + 4] / (bm_pix[0] *
                                                                   bm_pix[1])
                            p = N.insert(p,
                                         N.arange(len(fitfix)) * 6 + 6,
                                         total_flux[sind])
                            if sind > 0:
                                rms_img = img.__getattribute__('rms_' +
                                                               pols[sind] +
                                                               '_arr')
                            else:
                                rms_img = img.rms_arr
                            if len(rms_img.shape) > 1:
                                rms_isl = rms_img[isl.bbox].mean()
                            else:
                                rms_isl = rms_img
                            errors[sind] = func.get_errors(img, p, rms_isl)[6]

                    # Now, assign fluxes to each Gaussian.
                    src_flux_I = 0.0
                    src_flux_Q = 0.0
                    src_flux_U = 0.0
                    src_flux_V = 0.0
                    src_flux_I_err_sq = 0.0
                    src_flux_Q_err_sq = 0.0
                    src_flux_U_err_sq = 0.0
                    src_flux_V_err_sq = 0.0

                    for ig, gaussian in enumerate(src.gaussians):
                        flux_I = total_flux[0, ig]
                        flux_I_err = abs(errors[0, ig])
                        flux_Q = total_flux[1, ig]
                        flux_Q_err = abs(errors[1, ig])
                        flux_U = total_flux[2, ig]
                        flux_U_err = abs(errors[2, ig])
                        flux_V = total_flux[3, ig]
                        flux_V_err = abs(errors[3, ig])

                        if hasattr(src, '_pi'):
                            gaussian.total_flux = flux_I
                            gaussian.total_fluxE = flux_I_err
                        gaussian.total_flux_Q = flux_Q
                        gaussian.total_flux_U = flux_U
                        gaussian.total_flux_V = flux_V
                        gaussian.total_fluxE_Q = flux_Q_err
                        gaussian.total_fluxE_U = flux_U_err
                        gaussian.total_fluxE_V = flux_V_err

                        if hasattr(src, '_pi'):
                            src_flux_I += flux_I
                            src_flux_I_err_sq += flux_I_err**2
                        src_flux_Q += flux_Q
                        src_flux_U += flux_U
                        src_flux_V += flux_V
                        src_flux_Q_err_sq += flux_Q_err**2
                        src_flux_U_err_sq += flux_U_err**2
                        src_flux_V_err_sq += flux_V_err**2

                        # Calculate and store polarisation fractions and angle for each Gaussian in the island
                        # For this we need the I flux, which we can just take from g.total_flux and src.total_flux
                        flux_I = gaussian.total_flux
                        flux_I_err = gaussian.total_fluxE
                        stokes = [flux_I, flux_Q, flux_U, flux_V]
                        stokes_err = [
                            flux_I_err, flux_Q_err, flux_U_err, flux_V_err
                        ]

                        lpol_frac, lpol_frac_loerr, lpol_frac_hierr = self.calc_lpol_fraction(
                            stokes, stokes_err)  # linear pol fraction
                        lpol_ang, lpol_ang_err = self.calc_lpol_angle(
                            stokes, stokes_err)  # linear pol angle
                        cpol_frac, cpol_frac_loerr, cpol_frac_hierr = self.calc_cpol_fraction(
                            stokes, stokes_err)  # circular pol fraction
                        tpol_frac, tpol_frac_loerr, tpol_frac_hierr = self.calc_tpol_fraction(
                            stokes, stokes_err)  # total pol fraction

                        gaussian.lpol_fraction = lpol_frac
                        gaussian.lpol_fraction_loerr = lpol_frac_loerr
                        gaussian.lpol_fraction_hierr = lpol_frac_hierr
                        gaussian.cpol_fraction = cpol_frac
                        gaussian.cpol_fraction_loerr = cpol_frac_loerr
                        gaussian.cpol_fraction_hierr = cpol_frac_hierr
                        gaussian.tpol_fraction = tpol_frac
                        gaussian.tpol_fraction_loerr = tpol_frac_loerr
                        gaussian.tpol_fraction_hierr = tpol_frac_hierr
                        gaussian.lpol_angle = lpol_ang
                        gaussian.lpol_angle_err = lpol_ang_err

                    # Store fluxes for each source in the island
                    if hasattr(src, '_pi'):
                        src.total_flux = src_flux_I
                        src.total_fluxE = N.sqrt(src_flux_I_err_sq)
                    src.total_flux_Q = src_flux_Q
                    src.total_flux_U = src_flux_U
                    src.total_flux_V = src_flux_V
                    src.total_fluxE_Q = N.sqrt(src_flux_Q_err_sq)
                    src.total_fluxE_U = N.sqrt(src_flux_U_err_sq)
                    src.total_fluxE_V = N.sqrt(src_flux_V_err_sq)

                    # Calculate and store polarisation fractions and angle for each source in the island
                    # For this we need the I flux, which we can just take from g.total_flux and src.total_flux
                    src_flux_I = src.total_flux
                    src_flux_I_err = src.total_fluxE
                    stokes = [src_flux_I, src_flux_Q, src_flux_U, src_flux_V]
                    stokes_err = [
                        src_flux_I_err,
                        N.sqrt(src_flux_Q_err_sq),
                        N.sqrt(src_flux_U_err_sq),
                        N.sqrt(src_flux_V_err_sq)
                    ]

                    lpol_frac, lpol_frac_loerr, lpol_frac_hierr = self.calc_lpol_fraction(
                        stokes, stokes_err)  # linear pol fraction
                    lpol_ang, lpol_ang_err = self.calc_lpol_angle(
                        stokes, stokes_err)  # linear pol angle
                    cpol_frac, cpol_frac_loerr, cpol_frac_hierr = self.calc_cpol_fraction(
                        stokes, stokes_err)  # circular pol fraction
                    tpol_frac, tpol_frac_loerr, tpol_frac_hierr = self.calc_tpol_fraction(
                        stokes, stokes_err)  # total pol fraction

                    src.lpol_fraction = lpol_frac
                    src.lpol_fraction_loerr = lpol_frac_loerr
                    src.lpol_fraction_hierr = lpol_frac_hierr
                    src.cpol_fraction = cpol_frac
                    src.cpol_fraction_loerr = cpol_frac_loerr
                    src.cpol_fraction_hierr = cpol_frac_hierr
                    src.tpol_fraction = tpol_frac
                    src.tpol_fraction_loerr = tpol_frac_loerr
                    src.tpol_fraction_hierr = tpol_frac_hierr
                    src.lpol_angle = lpol_ang
                    src.lpol_angle_err = lpol_ang_err
                    if bar.started:
                        bar.increment()
            bar.stop()
            img.completed_Ops.append('polarisation')
Exemple #33
0
    def __call__(self, img):
        import time, os
        mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Readimage")

        if img.opts.filename == '':
            raise RuntimeError('Image file name not specified.')

        # Check for trailing "/" in file name (since CASA images are directories).
        # Although the general rule is to not alter the values in opts (only the
        # user should be able to alter these), in this case there is no harm in
        # replacing the file name in opts with the '/' trimmed off.
        if img.opts.filename[-1] == '/':
            img.opts.filename = img.opts.filename[:-1]

        # Determine indir if not explicitly given by user (in img.opts.indir)
        if img.opts.indir is None:
            indir = os.path.dirname(img.opts.filename)
            if indir == '':
                indir = './'
            img.indir = indir
        else:
            img.indir = img.opts.indir

        # Try to trim common extensions from filename and store various
        # paths
        root, ext = os.path.splitext(img.opts.filename)
        if ext in ['.fits', '.FITS', '.image']:
            fname = root
        elif ext in ['.gz', '.GZ']:
            root2, ext2 = os.path.splitext(root)
            if ext2 in ['.fits', '.FITS', '.image']:
                fname = root2
            else:
                fname = root
        else:
            fname = img.opts.filename
        img.filename = img.opts.filename
        img.parentname = fname
        img.imagename = fname + '.pybdsm'
        img.basedir = './' + fname + '_pybdsm/'

        # Read in data and header
        image_file = os.path.basename(img.opts.filename)
        result = read_image_from_file(image_file, img, img.indir)
        if result is None:
            raise RuntimeError("Cannot open file " + repr(image_file) + ". " +
                               img._reason)
        else:
            data, hdr = result

        # Check whether caching is to be used. If it is, set up a
        # temporary directory. The temporary directory will be
        # removed automatically upon exit.
        if img.opts.do_cache:
            img.do_cache = True
        else:
            img.do_cache = False
        if img.do_cache:
            mylog.info('Using disk caching.')
            tmpdir = img.parentname + '_tmp'
            if not os.path.exists(tmpdir):
                os.makedirs(tmpdir)
            img._tempdir_parent = TempDir(tmpdir)
            img.tempdir = TempDir(tempfile.mkdtemp(dir=tmpdir))
            import atexit, shutil
            atexit.register(shutil.rmtree,
                            img._tempdir_parent,
                            ignore_errors=True)
        else:
            img.tempdir = None

        # Store data and header in img. If polarisation_do = False, only store pol == 'I'
        img.nchan = data.shape[1]
        img.nstokes = data.shape[0]
        mylogger.userinfo(mylog, 'Image size',
                          str(data.shape[-2:]) + ' pixels')
        mylogger.userinfo(mylog, 'Number of channels', '%i' % data.shape[1])
        mylogger.userinfo(mylog, 'Number of Stokes parameters',
                          '%i' % data.shape[0])
        if img.opts.polarisation_do and data.shape[0] == 1:
            img.opts.polarisation_do = False
            mylog.warning(
                'Image has Stokes I only. Polarisation module disabled.')

        if img.opts.polarisation_do or data.shape[0] == 1:
            img.image_arr = data
        else:
            img.image_arr = data[0, :].reshape(1, data.shape[1], data.shape[2],
                                               data.shape[3])
        img.header = hdr
        img.shape = data.shape
        img.j = 0

        ### initialize wcs conversion routines
        self.init_wcs(img)
        self.init_beam(img)
        self.init_freq(img)
        year, code = self.get_equinox(img)
        if year is None:
            mylog.info('Equinox not found in image header. Assuming J2000.')
            img.equinox = 2000.0
        else:
            mylog.info('Equinox of image is %f.' % year)
            img.equinox = year

        if img.opts.output_all:
            # Set up directory to write output to
            opdir = img.opts.opdir_overwrite
            if opdir not in ['overwrite', 'append']:
                img.opts.opdir_overwrite = 'append'
            if opdir == 'append':
                mylog.info('Appending output files to directory ' +
                           img.basedir)
            else:
                mylog.info('Overwriting output files (if any) in directory ' +
                           img.basedir)
                if os.path.isdir(img.basedir):
                    os.system("rm -fr " + img.basedir + '/*')
            if not os.path.isdir(img.basedir):
                os.makedirs(img.basedir)

            # Now add solname (if any) and time to basedir
            if img.opts.solnname is not None:
                img.basedir += img.opts.solnname + '_'
            img.basedir += time.strftime("%d%b%Y_%H.%M.%S")

            # Make the final output directory
            if not os.path.isdir(img.basedir):
                os.makedirs(img.basedir)

        del data
        img.completed_Ops.append('readimage')
        return img
Exemple #34
0
    def __call__(self, img):
        global bar1

        mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"SpectIndex")
        img.mylog = mylog
        if img.opts.spectralindex_do:
            mylogger.userinfo(mylog, '\nExtracting spectral indices for all ch0 sources')
            shp = img.image_arr.shape
            if shp[1] > 1:
                # calc freq, beam_spectrum for nchan channels
                self.freq_beamsp_unav(img)
                sbeam = img.beam_spectrum
                freqin = img.freq

                # calc initial channel flags if needed
                iniflags = self.iniflag(img)
                img.specind_iniflags = iniflags
                good_chans = N.where(iniflags == False)
                unav_image = img.image_arr[0][good_chans]
                unav_freqs = freqin[good_chans]
                nmax_to_avg = img.opts.specind_maxchan
                nchan = unav_image.shape[0]
                mylog.info('After initial flagging of channels by rms, %i good channels remain' % (nchan,))
                if nmax_to_avg == 0:
                    nmax_to_avg = nchan

                # calculate the rms map of each unflagged channel
                bar1 = statusbar.StatusBar('Determing rms for channels in image ..... : ', 0, nchan)
                if img.opts.quiet == False:
                    bar1.start()
                rms_spec = self.rms_spectrum(img, unav_image) # bar1 updated here

                bar2 = statusbar.StatusBar('Calculating spectral indices for sources  : ', 0, img.nsrc)
                c_wts = img.opts.collapse_wt
                snr_desired = img.opts.specind_snr

                if img.opts.quiet == False and img.opts.verbose_fitting == False:
                    bar2.start()
                for src in img.sources:
                    isl = img.islands[src.island_id]
                    isl_bbox = isl.bbox

                    # Fit each channel with ch0 Gaussian(s) of the source,
                    # allowing only the normalization to vary.
                    chan_images = unav_image[:, isl_bbox[0], isl_bbox[1]]
                    chan_rms = rms_spec[:, isl_bbox[0], isl_bbox[1]]
                    beamlist = img.beam_spectrum
                    unavg_total_flux, e_unavg_total_flux = self.fit_channels(img, chan_images, chan_rms, src, beamlist)

                    # Check for upper limits and mask. gaus_mask is array of (N_channels x N_gaussians)
                    # and is True if measured flux is upper limit. n_good_chan_per_gaus is array of N_gaussians
                    # that gives number of unmasked channels for each Gaussian.
                    gaus_mask, n_good_chan_per_gaus = self.mask_upper_limits(unavg_total_flux, e_unavg_total_flux, snr_desired)

                    # Average if needed and fit again
                    # First find flux of faintest Gaussian of source and use it to estimate rms_desired
                    gflux = []
                    for g in src.gaussians:
                        gflux.append(g.peak_flux)
                    rms_desired = min(gflux)/snr_desired
                    total_flux = unavg_total_flux
                    e_total_flux = e_unavg_total_flux
                    freq_av = unav_freqs
                    nchan = chan_images.shape[0]
                    nchan_prev = nchan
                    while min(n_good_chan_per_gaus) < 2 and nchan > 2:
                        avimages, beamlist, freq_av, crms_av = self.windowaverage_cube(chan_images, rms_desired, chan_rms,
                                    c_wts, sbeam, freqin, nmax_to_avg=nmax_to_avg)
                        total_flux, e_total_flux = self.fit_channels(img, avimages, crms_av, src, beamlist)
                        gaus_mask, n_good_chan_per_gaus = self.mask_upper_limits(total_flux, e_total_flux, snr_desired)
                        nchan = avimages.shape[0]
                        if nchan == nchan_prev:
                            break
                        nchan_prev = nchan
                        rms_desired *= 0.8

                    # Now fit Gaussian fluxes to obtain spectral indices.
                    # Only fit if there are detections (at specified sigma threshold)
                    # in at least two bands. If not, don't fit and set spec_indx
                    # and error to NaN.
                    for ig, gaussian in enumerate(src.gaussians):
                        npos = len(N.where(total_flux[:, ig] > 0.0)[0])
                        if img.opts.verbose_fitting:
                            if img.opts.flagchan_snr:
                                print 'Gaussian #%i : averaged to %i channels, of which %i meet SNR criterion' % (gaussian.gaus_num,
                                       len(total_flux[:, ig]), n_good_chan_per_gaus[ig])
                            else:
                                print 'Gaussian #%i : averaged to %i channels, all of which will be used' % (gaussian.gaus_num,
                                       len(total_flux[:, ig]))
                        if (img.opts.flagchan_snr and n_good_chan_per_gaus[ig] < 2) or npos < 2:
                            gaussian.spec_indx = N.NaN
                            gaussian.e_spec_indx = N.NaN
                            gaussian.spec_norm = N.NaN
                            gaussian.specin_flux = [N.NaN]
                            gaussian.specin_fluxE = [N.NaN]
                            gaussian.specin_freq = [N.NaN]
                            gaussian.specin_freq0 = N.NaN
                        else:
                            if img.opts.flagchan_snr:
                                good_fluxes_ind = N.where(gaus_mask[:, ig] == False)
                            else:
                                good_fluxes_ind = range(len(freq_av))
                            fluxes_to_fit = total_flux[:, ig][good_fluxes_ind]
                            e_fluxes_to_fit = e_total_flux[:, ig][good_fluxes_ind]
                            freqs_to_fit = freq_av[good_fluxes_ind]
                            fit_res = self.fit_specindex(freqs_to_fit, fluxes_to_fit, e_fluxes_to_fit)
                            gaussian.spec_norm, gaussian.spec_indx, gaussian.e_spec_indx = fit_res
                            gaussian.specin_flux = fluxes_to_fit.tolist()
                            gaussian.specin_fluxE = e_fluxes_to_fit.tolist()
                            gaussian.specin_freq = freqs_to_fit.tolist()
                            gaussian.specin_freq0 = N.median(freqs_to_fit)

                    # Next fit total source fluxes for spectral index.
                    if len(src.gaussians) > 1:
                        # First, check unaveraged SNRs for total source.
                        src_total_flux = N.zeros((chan_images.shape[0], 1))
                        src_e_total_flux = N.zeros((chan_images.shape[0], 1))
                        src_total_flux[:,0] = N.sum(unavg_total_flux, 1) # sum over all Gaussians in source to obtain total fluxes in each channel
                        src_e_total_flux[:,0] = N.sqrt(N.sum(N.power(e_unavg_total_flux, 2.0), 1))
                        src_mask, n_good_chan = self.mask_upper_limits(src_total_flux, src_e_total_flux, snr_desired)

                        # Average if needed and fit again
                        rms_desired = src.peak_flux_max/snr_desired
                        total_flux = unavg_total_flux
                        e_total_flux = e_unavg_total_flux
                        freq_av = unav_freqs
                        nchan = chan_images.shape[0]
                        nchan_prev = nchan
                        while n_good_chan < 2 and nchan > 2:
                            avimages, beamlist, freq_av, crms_av = self.windowaverage_cube(chan_images, rms_desired, chan_rms,
                                        c_wts, sbeam, freqin, nmax_to_avg=nmax_to_avg)
                            total_flux, e_total_flux = self.fit_channels(img, avimages, crms_av, src, beamlist)
                            src_total_flux = N.sum(total_flux, 1) # sum over all Gaussians in source to obtain total fluxes in each channel
                            src_e_total_flux = N.sqrt(N.sum(N.power(e_total_flux, 2.0), 1))
                            src_mask, n_good_chan = self.mask_upper_limits(src_total_flux, src_e_total_flux, snr_desired)
                            nchan = avimages.shape[0]
                            if nchan == nchan_prev:
                                break
                            nchan_prev = nchan
                            rms_desired *= 0.8

                        # Now fit source for spectral index.
                        src_total_flux = src_total_flux.reshape((src_total_flux.shape[0],))
                        src_e_total_flux = src_e_total_flux.reshape((src_e_total_flux.shape[0],))
                        src_mask = src_mask.reshape((src_mask.shape[0],))
                        if img.opts.verbose_fitting:
                            if img.opts.flagchan_snr:
                                print 'Source #%i : averaged to %i channels, of which %i meet SNR criterion' % (src.source_id,
                                      len(src_total_flux), nchan)
                            else:
                                print 'Source #%i : averaged to %i channels, all of which will be used' % (src.source_id,
                                       len(src_total_flux))
                        npos = len(N.where(src_total_flux > 0.0)[0])

                        if isinstance(n_good_chan, int):
                            n_good_chan = [n_good_chan]
                        if (img.opts.flagchan_snr and n_good_chan[0] < 2) or npos < 2:
                            src.spec_indx = N.NaN
                            src.e_spec_indx = N.NaN
                            src.spec_norm = N.NaN
                            src.specin_flux = [N.NaN]
                            src.specin_fluxE = [N.NaN]
                            src.specin_freq = [N.NaN]
                            src.specin_freq0 = N.NaN
                        else:
                            if img.opts.flagchan_snr:
                                good_fluxes_ind = N.where(src_mask == False)
                            else:
                                good_fluxes_ind = range(len(freq_av))
                            fluxes_to_fit = src_total_flux[good_fluxes_ind]
                            e_fluxes_to_fit = src_e_total_flux[good_fluxes_ind]
                            freqs_to_fit = freq_av[good_fluxes_ind]

#                             if len(freqs_to_fit.shape) == 2:
#                                 freqs_to_fit = freqs_to_fit.reshape((freqs_to_fit.shape[0],))
#                             if len(fluxes_to_fit.shape) == 2:
#                                 fluxes_to_fit = fluxes_to_fit.reshape((fluxes_to_fit.shape[0],))
#                             if len(e_fluxes_to_fit.shape) == 2:
#                                 e_fluxes_to_fit = e_fluxes_to_fit.reshape((e_fluxes_to_fit.shape[0],))

                            fit_res = self.fit_specindex(freqs_to_fit, fluxes_to_fit, e_fluxes_to_fit)
                            src.spec_norm, src.spec_indx, src.e_spec_indx = fit_res
                            src.specin_flux = fluxes_to_fit.tolist()
                            src.specin_fluxE = e_fluxes_to_fit.tolist()
                            src.specin_freq = freqs_to_fit.tolist()
                            src.specin_freq0 = N.median(freqs_to_fit)
                    else:
                        src.spec_norm = src.gaussians[0].spec_norm
                        src.spec_indx = src.gaussians[0].spec_indx
                        src.e_spec_indx = src.gaussians[0].e_spec_indx
                        src.specin_flux = src.gaussians[0].specin_flux
                        src.specin_fluxE = src.gaussians[0].specin_fluxE
                        src.specin_freq = src.gaussians[0].specin_freq
                        src.specin_freq0 = src.gaussians[0].specin_freq0

                    if bar2.started:
                        bar2.increment()
                img.completed_Ops.append('spectralindex')
            else:
              mylog.warning('Image has only one channel. Spectral index module disabled.')
              img.opts.spectralindex_do = False
    def __call__(self, img):
        mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Preprocess")
        bstat = func.bstat
        if img.opts.kappa_clip is None:
            kappa = -img.pixel_beamarea()
        else:
            kappa = img.opts.kappa_clip

        if img.opts.polarisation_do:
            pols = ['I', 'Q', 'U', 'V']
            ch0images = [
                img.ch0_arr, img.ch0_Q_arr, img.ch0_U_arr, img.ch0_V_arr
            ]
            img.clipped_mean_QUV = []
            img.clipped_rms_QUV = []
        else:
            pols = ['I']  # assume I is always present
            ch0images = [img.ch0_arr]

        if hasattr(img, 'rms_mask'):
            mask = img.rms_mask
        else:
            mask = img.mask_arr
        opts = img.opts

        for ipol, pol in enumerate(pols):
            image = ch0images[ipol]

            ### basic stats
            mean, rms, cmean, crms, cnt = bstat(image, mask, kappa)
            if cnt > 198:
                cmean = mean
                crms = rms
            if pol == 'I':
                if func.approx_equal(crms, 0.0, rel=None):
                    raise RuntimeError('Clipped rms appears to be zero. Check for regions '\
                                           'with values of 0 and\nblank them (with NaNs) '\
                                           'or use trim_box to exclude them.')
                img.raw_mean = mean
                img.raw_rms = rms
                img.clipped_mean = cmean
                img.clipped_rms = crms
                mylog.info('%s %.4f %s %.4f %s ' % ("Raw mean (Stokes I) = ", mean*1000.0, \
                           'mJy and raw rms = ',rms*1000.0, 'mJy'))
                mylog.info('%s %.4f %s %s %.4f %s ' % ("sigma clipped mean (Stokes I) = ", cmean*1000.0, \
                           'mJy and ','sigma clipped rms = ',crms*1000.0, 'mJy'))
            else:
                img.clipped_mean_QUV.append(cmean)
                img.clipped_rms_QUV.append(crms)
                mylog.info('%s %s %s %.4f %s %s %.4f %s ' % ("sigma clipped mean (Stokes ", pol, ") = ", cmean*1000.0, \
                           'mJy and ','sigma clipped rms = ',crms*1000.0, 'mJy'))

        image = img.ch0_arr
        # Check if pixels are outside the universe
        if opts.check_outsideuniv:
            mylogger.userinfo(mylog,
                              "Checking for pixels outside the universe")
            noutside_univ = self.outside_univ(img)
            img.noutside_univ = noutside_univ
            frac_blank = round(
                float(noutside_univ) / float(image.shape[0] * image.shape[1]),
                3)
            mylogger.userinfo(
                mylog, "Number of additional pixels blanked",
                str(noutside_univ) + ' (' + str(frac_blank * 100.0) + '%)')
        else:
            noutside_univ = 0

        # If needed, (re)mask the image
        if noutside_univ > 0:
            mask = N.isnan(img.ch0_arr)
            masked = mask.any()
            img.masked = masked
            if masked:
                img.mask_arr = mask
            img.blankpix = N.sum(mask)

        ### max/min pixel value & coordinates
        shape = image.shape[0:2]
        if mask != None:
            img.blankpix = N.sum(mask)
        if img.blankpix == 0:
            max_idx = image.argmax()
            min_idx = image.argmin()
        else:
            max_idx = N.nanargmax(image)
            min_idx = N.nanargmin(image)

        img.maxpix_coord = N.unravel_index(max_idx, shape)
        img.minpix_coord = N.unravel_index(min_idx, shape)
        img.max_value = image.flat[max_idx]
        img.min_value = image.flat[min_idx]

        ### Solid angle of the image
        cdelt = N.array(img.wcs_obj.acdelt[:2])
        img.omega = N.product(shape) * abs(
            N.product(cdelt)) / (180. * 180. / pi / pi)

        ### Total flux in ch0 image
        if 'atrous' in img.filename or img._pi or img.log == 'Detection image':
            # Don't do this estimate for atrous wavelet images
            # or polarized intensity image,
            # as it doesn't give the correct flux. Also, ignore
            # the flux in the detection image, as it's likely
            # wrong (e.g., not corrected for the primary beam).
            img.ch0_sum_jy = 0
        else:
            im_flux = N.nansum(image) / img.pixel_beamarea()  # Jy
            img.ch0_sum_jy = im_flux
            mylogger.userinfo(mylog, 'Flux from sum of (non-blank) pixels',
                              '%.3f Jy' % (im_flux, ))

        ### if image seems confused, then take background mean as zero instead
        alpha_sourcecounts = 2.5  # approx diff src count slope. 2.2?
        if opts.bmpersrc_th is None:
            n = (image >= 5. * crms).sum()
            if n <= 0:
                n = 1
                mylog.info('No pixels in image > 5-sigma.')
                mylog.info('Taking number of pixels above 5-sigma as 1.')
            img.bmpersrc_th = N.product(shape) / (
                (alpha_sourcecounts - 1.) * n)
            mylog.info('%s %6.2f' %
                       ('Estimated bmpersrc_th = ', img.bmpersrc_th))
        else:
            img.bmpersrc_th = opts.bmpersrc_th
            mylog.info('%s %6.2f' %
                       ('Taking default bmpersrc_th = ', img.bmpersrc_th))

        confused = False
        if opts.mean_map == 'default':
            if opts.bmpersrc_th <= 25. or cmean / crms >= 0.1:
                confused = True
        img.confused = confused
        mylog.info('Parameter confused is ' + str(img.confused))

        img.completed_Ops.append('preprocess')
        return img
Exemple #36
0
    def init_beam(self, img):
        """Initialize beam parameters, and conversion routines
        to convert beam to/from pixel coordinates"""
        from const import fwsig
        mylog = mylogger.logging.getLogger("PyBDSM.InitBeam")

        hdr = img.header
        cdelt1, cdelt2 = img.wcs_obj.acdelt[0:2]

        ### define beam conversion routines:
        def beam2pix(x):
            """ Converts beam in deg to pixels. Use when no dependence on
            position is appropriate.

            Input beam angle should be degrees CCW from North at image center.
            The output beam angle is degrees CCW from the +y axis of the image.
            """
            bmaj, bmin, bpa = x
            s1 = abs(bmaj / cdelt1)
            s2 = abs(bmin / cdelt2)
            th = bpa
            return (s1, s2, th)

        def pix2beam(x):
            """ Converts beam in pixels to deg. Use when no dependence on
            position is appropriate.

            Input beam angle should be degrees CCW from the +y axis of the image.
            The output beam angle is degrees CCW from North at image center.
            """
            s1, s2, th = x
            bmaj = abs(s1 * cdelt1)
            bmin = abs(s2 * cdelt2)
            bpa = th
            if bmaj < bmin:
                bmaj, bmin = bmin, bmaj
                bpa += 90.0
            bpa = divmod(bpa, 180)[1]  ### bpa lies between 0 and 180
            return [bmaj, bmin, bpa]

        def pixel_beam():
            """Returns the beam in sigma units in pixels"""
            pbeam = beam2pix(img.beam)
            return (pbeam[0] / fwsig, pbeam[1] / fwsig, pbeam[2])

        def pixel_beamarea():
            """Returns the beam area in pixels"""
            pbeam = beam2pix(img.beam)
            return 1.1331 * pbeam[0] * pbeam[1]

        ### Get the beam information from the header
        found = False
        if img.opts.beam is not None:
            beam = img.opts.beam
        else:
            try:
                beam = (hdr['BMAJ'], hdr['BMIN'], hdr['BPA'])
                found = True
            except:
                ### try see if AIPS as put the beam in HISTORY as usual
                for h in hdr['HISTORY']:
                    # Check if h is a string or a FITS Card object (long headers are
                    # split into Cards as of PyFITS 3.0.4)
                    if not isinstance(h, str):
                        hstr = h.value
                    else:
                        hstr = h
                    if N.all([
                            'BMAJ' in hstr, 'BMIN' in hstr, 'BPA' in hstr,
                            'CLEAN' in hstr
                    ]):
                        try:
                            dum, dum, dum, bmaj, dum, bmin, dum, bpa = hstr.split(
                            )
                        except ValueError:
                            try:
                                dum, dum, bmaj, dum, bmin, dum, bpa, dum, dum = hstr.split(
                                )
                            except ValueError:
                                break
                        beam = (float(bmaj), float(bmin), float(bpa))
                        found = True
            if not found:
                raise RuntimeError(
                    "No beam information found in image header.")

        ### convert beam into pixels (at image center)
        pbeam = beam2pix(beam)
        pbeam = (pbeam[0] / fwsig, pbeam[1] / fwsig, pbeam[2]
                 )  # IN SIGMA UNITS

        ### and store it
        img.pix2beam = pix2beam
        img.beam2pix = beam2pix
        img.beam = beam  # FWHM size in degrees
        img.pixel_beam = pixel_beam  # IN SIGMA UNITS in pixels
        img.pixel_beamarea = pixel_beamarea
        mylogger.userinfo(
            mylog, 'Beam shape (major, minor, pos angle)',
            '(%.5e, %.5e, %s) degrees' % (beam[0], beam[1], round(beam[2], 1)))
Exemple #37
0
    def calculate_maps(self,
                       img,
                       data,
                       mean,
                       rms,
                       mask,
                       map_opts,
                       do_adapt,
                       bright_pt_coords=[],
                       rms_box2=None,
                       logname=None,
                       ncores=None):
        """Calls map_2d and checks for problems"""
        mylog = mylogger.logging.getLogger("PyBDSM." + img.log +
                                           "Rmsimage.Calcmaps ")
        rms_ok = False
        mylog = mylogger.logging.getLogger("PyBDSM." + img.log +
                                           "Rmsimage.Calcmaps ")
        opts = img.opts
        kappa = map_opts[0]
        while not rms_ok:
            self.map_2d(data,
                        mean,
                        rms,
                        mask,
                        *map_opts,
                        do_adapt=do_adapt,
                        bright_pt_coords=bright_pt_coords,
                        rms_box2=rms_box2,
                        logname=logname,
                        ncores=ncores)
            if N.any(rms < 0.0):
                rms_ok = False
                if (opts.rms_box_bright is None
                        and do_adapt) or (opts.rms_box is None
                                          and not do_adapt):
                    # Increase box by 20%
                    if do_adapt:
                        new_width = int(img.rms_box_bright[0] * 1.2)
                        if new_width == img.rms_box_bright[0]:
                            new_width = img.rms_box_bright[0] + 1
                        new_step = int(new_width / 3.0)
                        img.rms_box_bright = (new_width, new_step)
                        if img.rms_box_bright[0] > min(
                                img.ch0_arr.shape) / 4.0:
                            mylogger.userinfo(
                                mylog,
                                'Size of rms_box_bright larger than 1/4 of image size'
                            )
                            mylogger.userinfo(
                                mylog,
                                'Using constant background rms and mean')
                            img.use_rms_map = False
                            img.rms_box = img.rms_box_bright
                            img.mean_map_type = 'const'
                            rms_ok = True
                        else:
                            map_opts = (kappa, img.rms_box_bright,
                                        opts.spline_rank)
                    else:
                        new_width = int(img.rms_box[0] * 1.2)
                        if new_width == img.rms_box[0]:
                            new_width = img.rms_box[0] + 1
                        new_step = int(new_width / 3.0)
                        img.rms_box = (new_width, new_step)
                        if img.rms_box[0] > min(img.ch0_arr.shape) / 4.0:
                            mylogger.userinfo(
                                mylog,
                                'Size of rms_box larger than 1/4 of image size'
                            )
                            mylogger.userinfo(
                                mylog,
                                'Using constant background rms and mean')
                            img.use_rms_map = False
                            img.mean_map_type = 'const'
                            rms_ok = True
                        else:
                            map_opts = (kappa, img.rms_box, opts.spline_rank)

                else:
                    # User has specified box size, use order=1 to prevent negatives
                    if opts.spline_rank > 1:
                        mylog.warning(
                            'Negative values found in rms map interpolated with spline_rank = %i'
                            % opts.spline_rank)
                        mylog.warning(
                            'Using spline_rank = 1 (bilinear interpolation) instead'
                        )
                        if do_adapt:
                            map_opts = (kappa, img.rms_box_bright, 1)
                        else:
                            map_opts = (kappa, img.rms_box, 1)
            else:
                rms_ok = True

        return mean, rms
Exemple #38
0
    def __call__(self, img):
        mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Islands")
        opts = img.opts

        minsize = opts.minpix_isl
        if minsize is None:
            minsize = int(img.pixel_beamarea() /
                          3.0)  # 1/3 of beam area in pixels
            if minsize < 6:
                minsize = 6  # Need at least 6 pixels to obtain good fits
            mylogger.userinfo(mylog, "Minimum number of pixels per island",
                              '%i' % minsize)
        img.minpix_isl = minsize

        if opts.detection_image != '':
            # Use a different image for island detection. The detection
            # image and the measurement image must have the same shape
            # and be registered. Otherwise, one could reproject the
            # detection image using, e.g., the Kapteyn package.
            #
            # First, set up up an Image object and run a limited
            # op_chain.
            from . import _run_op_list
            mylogger.userinfo(mylog,
                              "\nDetermining islands from detection image")

            det_chain, det_opts = self.setpara_bdsm(img, opts.detection_image)
            det_img = Image(det_opts)
            det_img.log = 'Detection image'
            success = _run_op_list(det_img, det_chain)
            if not success:
                return

            # Check that the ch0 images are the same size
            ch0_map = img.ch0_arr
            det_ch0_map = det_img.ch0_arr
            det_shape = det_ch0_map.shape
            ch0_shape = ch0_map.shape
            if det_shape != ch0_shape:
                raise RuntimeError(
                    "Detection image shape does not match that of input image."
                )

            # Run through islands and correct the image and rms, mean and max values
            img.island_labels = det_img.island_labels
            corr_islands = []
            mean_map = img.mean_arr
            rms_map = img.rms_arr
            for i, isl in enumerate(det_img.islands):
                islcp = isl.copy(img.pixel_beamarea(),
                                 image=ch0_map[isl.bbox],
                                 mean=mean_map[isl.bbox],
                                 rms=rms_map[isl.bbox])
                islcp.island_id = i
                corr_islands.append(islcp)
            img.islands = corr_islands
            img.nisl = len(img.islands)
            img.pyrank = det_img.pyrank
            img.minpix_isl = det_img.minpix_isl
            mylogger.userinfo(mylog,
                              "\nContinuing processing using primary image")
        else:
            if opts.src_ra_dec is not None:
                mylogger.userinfo(
                    mylog,
                    "Constructing islands at user-supplied source locations")
                img.islands = self.coords_to_isl(img, opts)
            else:
                img.islands = self.ndimage_alg(img, opts)
            img.nisl = len(img.islands)

            mylogger.userinfo(mylog, "Number of islands found",
                              '%i' % len(img.islands))

            ch0_map = img.ch0_arr
            ch0_shape = ch0_map.shape
            pyrank = N.zeros(ch0_shape, dtype=N.int32)
            for i, isl in enumerate(img.islands):
                isl.island_id = i
                pyrank[isl.bbox] += N.invert(isl.mask_active) * (i + 1)
            pyrank -= 1  # align pyrank values with island ids and set regions outside of islands to -1

            if opts.output_all: write_islands(img)
            if opts.savefits_rankim:
                func.write_image_to_file(img.use_io,
                                         img.imagename + '_pyrank.fits',
                                         pyrank, img)

            img.pyrank = pyrank

        img.completed_Ops.append('islands')
        return img
Exemple #39
0
    def __call__(self, img):
        mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Polarisatn")
        if img.opts.polarisation_do:
          mylog.info('Extracting polarisation properties for all sources')
          pols = ['I', 'Q', 'U', 'V']

          # Run gausfit and gual2srl on PI image to look for polarized sources
          # undetected in I
          fit_PI = img.opts.pi_fit
          n_new = 0
          ch0_pi = N.sqrt(img.ch0_Q_arr**2 + img.ch0_U_arr**2)
          img.ch0_pi_arr = ch0_pi

          if fit_PI:
              from . import _run_op_list
              mylogger.userinfo(mylog, "\nChecking PI image for new sources")

              mask = img.mask_arr
              minsize = img.opts.minpix_isl

              # Set up image object for PI image.
              pi_chain, pi_opts = self.setpara_bdsm(img)
              pimg = Image(pi_opts)
              pimg.beam = img.beam
              pimg.pixel_beam = img.pixel_beam
              pimg.pixel_beamarea = img.pixel_beamarea
              pimg.log = 'PI.'
              pimg.pix2beam = img.pix2beam
              pimg.beam2pix = img.beam2pix
              pimg.pix2gaus = img.pix2gaus
              pimg.gaus2pix = img.gaus2pix
              pimg.pix2sky = img.pix2sky
              pimg.sky2pix = img.sky2pix
              pimg.pix2coord = img.pix2coord
              pimg.wcs_obj = img.wcs_obj
              pimg.mask_arr = mask
              pimg.masked = img.masked
              pimg.ch0_arr = ch0_pi
              pimg._pi = True

              success = _run_op_list(pimg, pi_chain)
              if not success:
                  return

              img.pi_islands = pimg.islands
              img.pi_gaussians = pimg.gaussians
              img.pi_sources = pimg.sources

              # Now check for new sources in the PI image that are not
              # found in the Stokes I image. If any new sources are found,
              # adjust their IDs to follow after those found in I.
              new_isl = []
              new_src = []
              new_gaus = []
              n_new_src = 0
              isl_id = img.islands[-1].island_id
              src_id = img.sources[-1].source_id
              gaus_id = img.gaussians[-1].gaus_num
              for pi_isl in pimg.islands:
                  new_sources = []
                  for pi_src in pi_isl.sources:
                      if img.pyrank[int(img.sky2pix(pi_src.posn_sky_max)[0]),
                                    int(img.sky2pix(pi_src.posn_sky_max)[1])] == -1:
                          src_id += 1
                          pi_src._pi = True
                          pi_src.island_id = isl_id
                          pi_src.source_id = src_id
                          pi_src.spec_indx = N.NaN
                          pi_src.e_spec_indx = N.NaN
                          pi_src.spec_norm = N.NaN
                          pi_src.specin_flux = [N.NaN]
                          pi_src.specin_fluxE = [N.NaN]
                          pi_src.specin_freq = [N.NaN]
                          pi_src.specin_freq0 = N.NaN
                          new_sources.append(pi_src)
                          new_src.append(pi_src)
                          n_new_src += 1
                          for g in pi_src.gaussians:
                              gaus_id += 1
                              new_gaus.append(g)
                              g.gaus_num = gaus_id
                  if len(new_sources) > 0:
                      isl_id += 1
                      pi_isl.sources = new_sources
                      pi_isl.island_id = isl_id
                      pi_isl._pi = True
                      new_isl.append(pi_isl)

              n_new = len(new_isl)
              mylogger.userinfo(mylog, "New sources found in PI image", '%i (%i total)' %
                                (n_new, img.nsrc+n_new))

          if n_new > 0:
              img.islands += new_isl
              img.sources += new_src
              img.gaussians += new_gaus
              img.nsrc += n_new_src

          bar = statusbar.StatusBar('Calculating polarisation properties ....  : ', 0, img.nsrc)
          if img.opts.quiet == False:
              bar.start()

          for isl in img.islands:
            isl_bbox = isl.bbox
            ch0_I = img.ch0_arr[isl_bbox]
            ch0_Q = img.ch0_Q_arr[isl_bbox]
            ch0_U = img.ch0_U_arr[isl_bbox]
            ch0_V = img.ch0_V_arr[isl_bbox]
            ch0_images = [ch0_I, ch0_Q, ch0_U, ch0_V]

            for i, src in enumerate(isl.sources):
                # For each source, assume the morphology does not change
                # across the Stokes cube. This assumption allows us to fit
                # the Gaussians of each source to each Stokes image by
                # simply fitting only the overall normalizations of the
                # individual Gaussians.
                #
                # First, fit all source Gaussians to each Stokes image:
                x, y = N.mgrid[isl_bbox]
                gg = src.gaussians
                fitfix = N.ones(len(gg)) # fit only normalization
                srcmask = isl.mask_active
                total_flux = N.zeros((4, len(fitfix)), dtype=N.float32) # array of fluxes: N_Stokes x N_Gaussians
                errors = N.zeros((4, len(fitfix)), dtype=N.float32) # array of fluxes: N_Stokes x N_Gaussians

                for sind, image in enumerate(ch0_images):
                    if (sind==0 and hasattr(src, '_pi')) or sind > 0: # Fit I only for PI sources
                        p, ep = func.fit_mulgaus2d(image, gg, x, y, srcmask, fitfix)
                        for ig in range(len(fitfix)):
                            center_pix = (p[ig*6 + 1], p[ig*6 + 2])
                            bm_pix = N.array([img.pixel_beam()[0], img.pixel_beam()[1], img.pixel_beam()[2]])
                            total_flux[sind, ig] = p[ig*6]*p[ig*6+3]*p[ig*6+4]/(bm_pix[0]*bm_pix[1])
                        p = N.insert(p, N.arange(len(fitfix))*6+6, total_flux[sind])
                        if sind > 0:
                            rms_img = img.__getattribute__('rms_'+pols[sind]+'_arr')
                        else:
                            rms_img = img.rms_arr
                        if len(rms_img.shape) > 1:
                            rms_isl = rms_img[isl.bbox].mean()
                        else:
                            rms_isl = rms_img
                        errors[sind] = func.get_errors(img, p, rms_isl)[6]

                # Now, assign fluxes to each Gaussian.
                src_flux_I = 0.0
                src_flux_Q = 0.0
                src_flux_U = 0.0
                src_flux_V = 0.0
                src_flux_I_err_sq = 0.0
                src_flux_Q_err_sq = 0.0
                src_flux_U_err_sq = 0.0
                src_flux_V_err_sq = 0.0

                for ig, gaussian in enumerate(src.gaussians):
                    flux_I = total_flux[0, ig]
                    flux_I_err = abs(errors[0, ig])
                    flux_Q = total_flux[1, ig]
                    flux_Q_err = abs(errors[1, ig])
                    flux_U = total_flux[2, ig]
                    flux_U_err = abs(errors[2, ig])
                    flux_V = total_flux[3, ig]
                    flux_V_err = abs(errors[3, ig])

                    if hasattr(src, '_pi'):
                        gaussian.total_flux = flux_I
                        gaussian.total_fluxE = flux_I_err
                    gaussian.total_flux_Q = flux_Q
                    gaussian.total_flux_U = flux_U
                    gaussian.total_flux_V = flux_V
                    gaussian.total_fluxE_Q = flux_Q_err
                    gaussian.total_fluxE_U = flux_U_err
                    gaussian.total_fluxE_V = flux_V_err

                    if hasattr(src, '_pi'):
                        src_flux_I += flux_I
                        src_flux_I_err_sq += flux_I_err**2
                    src_flux_Q += flux_Q
                    src_flux_U += flux_U
                    src_flux_V += flux_V
                    src_flux_Q_err_sq += flux_Q_err**2
                    src_flux_U_err_sq += flux_U_err**2
                    src_flux_V_err_sq += flux_V_err**2

                    # Calculate and store polarisation fractions and angle for each Gaussian in the island
                    # For this we need the I flux, which we can just take from g.total_flux and src.total_flux
                    flux_I = gaussian.total_flux
                    flux_I_err = gaussian.total_fluxE
                    stokes = [flux_I, flux_Q, flux_U, flux_V]
                    stokes_err = [flux_I_err, flux_Q_err, flux_U_err, flux_V_err]

                    lpol_frac, lpol_frac_loerr, lpol_frac_hierr = self.calc_lpol_fraction(stokes, stokes_err) # linear pol fraction
                    lpol_ang, lpol_ang_err = self.calc_lpol_angle(stokes, stokes_err) # linear pol angle
                    cpol_frac, cpol_frac_loerr, cpol_frac_hierr = self.calc_cpol_fraction(stokes, stokes_err) # circular pol fraction
                    tpol_frac, tpol_frac_loerr, tpol_frac_hierr = self.calc_tpol_fraction(stokes, stokes_err) # total pol fraction

                    gaussian.lpol_fraction = lpol_frac
                    gaussian.lpol_fraction_loerr = lpol_frac_loerr
                    gaussian.lpol_fraction_hierr = lpol_frac_hierr
                    gaussian.cpol_fraction = cpol_frac
                    gaussian.cpol_fraction_loerr = cpol_frac_loerr
                    gaussian.cpol_fraction_hierr = cpol_frac_hierr
                    gaussian.tpol_fraction = tpol_frac
                    gaussian.tpol_fraction_loerr = tpol_frac_loerr
                    gaussian.tpol_fraction_hierr = tpol_frac_hierr
                    gaussian.lpol_angle = lpol_ang
                    gaussian.lpol_angle_err = lpol_ang_err

                # Store fluxes for each source in the island
                if hasattr(src, '_pi'):
                    src.total_flux = src_flux_I
                    src.total_fluxE = N.sqrt(src_flux_I_err_sq)
                src.total_flux_Q = src_flux_Q
                src.total_flux_U = src_flux_U
                src.total_flux_V = src_flux_V
                src.total_fluxE_Q = N.sqrt(src_flux_Q_err_sq)
                src.total_fluxE_U = N.sqrt(src_flux_U_err_sq)
                src.total_fluxE_V = N.sqrt(src_flux_V_err_sq)

                # Calculate and store polarisation fractions and angle for each source in the island
                # For this we need the I flux, which we can just take from g.total_flux and src.total_flux
                src_flux_I = src.total_flux
                src_flux_I_err = src.total_fluxE
                stokes = [src_flux_I, src_flux_Q, src_flux_U, src_flux_V]
                stokes_err = [src_flux_I_err, N.sqrt(src_flux_Q_err_sq), N.sqrt(src_flux_U_err_sq), N.sqrt(src_flux_V_err_sq)]

                lpol_frac, lpol_frac_loerr, lpol_frac_hierr = self.calc_lpol_fraction(stokes, stokes_err) # linear pol fraction
                lpol_ang, lpol_ang_err = self.calc_lpol_angle(stokes, stokes_err) # linear pol angle
                cpol_frac, cpol_frac_loerr, cpol_frac_hierr = self.calc_cpol_fraction(stokes, stokes_err) # circular pol fraction
                tpol_frac, tpol_frac_loerr, tpol_frac_hierr = self.calc_tpol_fraction(stokes, stokes_err) # total pol fraction

                src.lpol_fraction = lpol_frac
                src.lpol_fraction_loerr = lpol_frac_loerr
                src.lpol_fraction_hierr = lpol_frac_hierr
                src.cpol_fraction = cpol_frac
                src.cpol_fraction_loerr = cpol_frac_loerr
                src.cpol_fraction_hierr = cpol_frac_hierr
                src.tpol_fraction = tpol_frac
                src.tpol_fraction_loerr = tpol_frac_loerr
                src.tpol_fraction_hierr = tpol_frac_hierr
                src.lpol_angle = lpol_ang
                src.lpol_angle_err = lpol_ang_err
                if bar.started:
                    bar.increment()
          bar.stop()
          img.completed_Ops.append('polarisation')