def test_status_formatting(self): sb = statusbar.StatusBar("Test") sb.add_progress(2, '#') sb.add_progress(2, '.') result = sb.format_status(15) # -- I don't actually know how to get the visible string length # self.assertEqual(len(result), 15) sb.set_progress_brackets("", "") result = sb.format_status(15) # self.assertEqual(len(result), 15) sb = statusbar.StatusBar("Long label") sb.add_progress(2, '#') sb.add_progress(2, '.') result = sb.format_status(15, label_width=4) # self.assertEqual(len(result), 15) sb = statusbar.StatusBar("Long label") sb.add_progress(2, '#') sb.add_progress(2, '.') result = sb.format_status(26, label_width=15) # self.assertEqual(len(result), 26) result = sb.format_status(label_width=4, progress_width=10) result = sb.format_status(label_width=4, progress_width=10, summary_width=5) self.assertEqual(len(result), 35) # this is only to avoid lint issues
def test_formatting_with_fill_char(self): sb = statusbar.StatusBar("Test", fill_char=' ') sb.add_progress(1, '#') result = sb.format_status(15, label_width=5) self.assertTrue(result.startswith('Test ')) sb = statusbar.StatusBar("Test", fill_char='_') sb.add_progress(1, '#') result = sb.format_status(15, label_width=5) self.assertTrue(result.startswith('Test_ '))
def __init__(self, needs_render_window): """Constructor. Initializes the application main window and restores the previous run. """ wx.MDIParentFrame.__init__(self, None, -1, "Conjurer GUI", size=(1024, 768)) # window icon file_server = servers.get_file_server() icon_path = file_server.manglepath("outgui:images/conjurer.ico") self.SetIcon(wx.Icon(icon_path, wx.BITMAP_TYPE_ICO)) # Let the InGUI know that the OutGUI is open pynebula.new('nroot', '/editor/outguiisopened') self.quit_without_saving = False self.quit_requested = False # create a render window if required if needs_render_window: self.render_window = renderwindow.CreateWindow(self) parent_hwnd_env = pynebula.new('nenv', '/sys/env/hwnd') parent_hwnd_env.seti(self.render_window.GetPanelHandle()) else: self.render_window = None # preparing repository nebulaguisettings.Repository # toggable window manager self.togwinmgr = togwin.ToggableWindowMgr(self) # menu bar self.__menubar = menubar.MenuBar(self) self.__menubar.create() self.SetMenuBar(self.__menubar) # status bar self.__statusbar = statusbar.StatusBar(self) self.__statusbar.create() self.SetStatusBar(self.__statusbar) # tool bar self.__toolbar = toolbar.ToolBar(self) self.__toolbar.create() self.SetToolBar(self.__toolbar) self.__toolbar.Realize() # events self.Bind(wx.EVT_CLOSE, self.on_close_window) self.Bind(nh.EVT_CONJURER_HELP, self.on_conjurer_help) # TODO: Delete when found a direct way to know the mouse position self.__mouse_position = wx.Point(0, 0) self.__toolbar.Bind(wx.EVT_LEFT_UP, self.__on_left_click)
def __call__(self, img): mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Shapefit") bar = statusbar.StatusBar( 'Decomposing islands into shapelets ...... : ', 0, img.nisl) opts = img.opts if img.opts.shapelet_do: if opts.quiet == False: bar.start() # Set up multiproccessing. First create a simple copy of the Image # object that contains the minimal data needed. opts_dict = opts.to_dict() img_simple = Image(opts_dict) img_simple.pixel_beamarea = img.pixel_beamarea img_simple.pixel_beam = img.pixel_beam img_simple.thresh_pix = img.thresh_pix img_simple.minpix_isl = img.minpix_isl img_simple.clipped_mean = img.clipped_mean img_simple.shape = img.ch0_arr.shape # Now call the parallel mapping function. Returns a list of # [beta, centre, nmax, basis, cf] for each island shap_list = mp.parallel_map( func.eval_func_tuple, itertools.izip(itertools.repeat(self.process_island), img.islands, itertools.repeat(img_simple), itertools.repeat(opts)), numcores=opts.ncores, bar=bar) for id, isl in enumerate(img.islands): beta, centre, nmax, basis, cf = shap_list[id] isl.shapelet_beta = beta isl.shapelet_centre = centre isl.shapelet_posn_sky = img.pix2sky(centre) isl.shapelet_posn_skyE = [0.0, 0.0, 0.0] isl.shapelet_nmax = nmax isl.shapelet_basis = basis isl.shapelet_cf = cf img.completed_Ops.append('shapelets')
def __call__(self, img): global bar1 mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"SpectIndex") img.mylog = mylog if img.opts.spectralindex_do: mylogger.userinfo(mylog, '\nExtracting spectral indices for all ch0 sources') shp = img.image_arr.shape if shp[1] > 1: # calc freq, beam_spectrum for nchan channels self.freq_beamsp_unav(img) sbeam = img.beam_spectrum freqin = img.freq # calc initial channel flags if needed iniflags = self.iniflag(img) img.specind_iniflags = iniflags good_chans = N.where(iniflags == False) unav_image = img.image_arr[0][good_chans] unav_freqs = freqin[good_chans] nmax_to_avg = img.opts.specind_maxchan nchan = unav_image.shape[0] mylog.info('After initial flagging of channels by rms, %i good channels remain' % (nchan,)) if nmax_to_avg == 0: nmax_to_avg = nchan # calculate the rms map of each unflagged channel bar1 = statusbar.StatusBar('Determing rms for channels in image ..... : ', 0, nchan) if img.opts.quiet == False: bar1.start() rms_spec = self.rms_spectrum(img, unav_image) # bar1 updated here bar2 = statusbar.StatusBar('Calculating spectral indices for sources : ', 0, img.nsrc) c_wts = img.opts.collapse_wt snr_desired = img.opts.specind_snr if img.opts.quiet == False and img.opts.verbose_fitting == False: bar2.start() for src in img.sources: isl = img.islands[src.island_id] isl_bbox = isl.bbox # Fit each channel with ch0 Gaussian(s) of the source, # allowing only the normalization to vary. chan_images = unav_image[:, isl_bbox[0], isl_bbox[1]] chan_rms = rms_spec[:, isl_bbox[0], isl_bbox[1]] beamlist = img.beam_spectrum unavg_total_flux, e_unavg_total_flux = self.fit_channels(img, chan_images, chan_rms, src, beamlist) # Check for upper limits and mask. gaus_mask is array of (N_channels x N_gaussians) # and is True if measured flux is upper limit. n_good_chan_per_gaus is array of N_gaussians # that gives number of unmasked channels for each Gaussian. gaus_mask, n_good_chan_per_gaus = self.mask_upper_limits(unavg_total_flux, e_unavg_total_flux, snr_desired) # Average if needed and fit again # First find flux of faintest Gaussian of source and use it to estimate rms_desired gflux = [] for g in src.gaussians: gflux.append(g.peak_flux) rms_desired = min(gflux)/snr_desired total_flux = unavg_total_flux e_total_flux = e_unavg_total_flux freq_av = unav_freqs nchan = chan_images.shape[0] nchan_prev = nchan while min(n_good_chan_per_gaus) < 2 and nchan > 2: avimages, beamlist, freq_av, crms_av = self.windowaverage_cube(chan_images, rms_desired, chan_rms, c_wts, sbeam, freqin, nmax_to_avg=nmax_to_avg) total_flux, e_total_flux = self.fit_channels(img, avimages, crms_av, src, beamlist) gaus_mask, n_good_chan_per_gaus = self.mask_upper_limits(total_flux, e_total_flux, snr_desired) nchan = avimages.shape[0] if nchan == nchan_prev: break nchan_prev = nchan rms_desired *= 0.8 # Now fit Gaussian fluxes to obtain spectral indices. # Only fit if there are detections (at specified sigma threshold) # in at least two bands. If not, don't fit and set spec_indx # and error to NaN. for ig, gaussian in enumerate(src.gaussians): npos = len(N.where(total_flux[:, ig] > 0.0)[0]) if img.opts.verbose_fitting: if img.opts.flagchan_snr: print 'Gaussian #%i : averaged to %i channels, of which %i meet SNR criterion' % (gaussian.gaus_num, len(total_flux[:, ig]), n_good_chan_per_gaus[ig]) else: print 'Gaussian #%i : averaged to %i channels, all of which will be used' % (gaussian.gaus_num, len(total_flux[:, ig])) if (img.opts.flagchan_snr and n_good_chan_per_gaus[ig] < 2) or npos < 2: gaussian.spec_indx = N.NaN gaussian.e_spec_indx = N.NaN gaussian.spec_norm = N.NaN gaussian.specin_flux = [N.NaN] gaussian.specin_fluxE = [N.NaN] gaussian.specin_freq = [N.NaN] gaussian.specin_freq0 = N.NaN else: if img.opts.flagchan_snr: good_fluxes_ind = N.where(gaus_mask[:, ig] == False) else: good_fluxes_ind = range(len(freq_av)) fluxes_to_fit = total_flux[:, ig][good_fluxes_ind] e_fluxes_to_fit = e_total_flux[:, ig][good_fluxes_ind] freqs_to_fit = freq_av[good_fluxes_ind] fit_res = self.fit_specindex(freqs_to_fit, fluxes_to_fit, e_fluxes_to_fit) gaussian.spec_norm, gaussian.spec_indx, gaussian.e_spec_indx = fit_res gaussian.specin_flux = fluxes_to_fit.tolist() gaussian.specin_fluxE = e_fluxes_to_fit.tolist() gaussian.specin_freq = freqs_to_fit.tolist() gaussian.specin_freq0 = N.median(freqs_to_fit) # Next fit total source fluxes for spectral index. if len(src.gaussians) > 1: # First, check unaveraged SNRs for total source. src_total_flux = N.zeros((chan_images.shape[0], 1)) src_e_total_flux = N.zeros((chan_images.shape[0], 1)) src_total_flux[:,0] = N.sum(unavg_total_flux, 1) # sum over all Gaussians in source to obtain total fluxes in each channel src_e_total_flux[:,0] = N.sqrt(N.sum(N.power(e_unavg_total_flux, 2.0), 1)) src_mask, n_good_chan = self.mask_upper_limits(src_total_flux, src_e_total_flux, snr_desired) # Average if needed and fit again rms_desired = src.peak_flux_max/snr_desired total_flux = unavg_total_flux e_total_flux = e_unavg_total_flux freq_av = unav_freqs nchan = chan_images.shape[0] nchan_prev = nchan while n_good_chan < 2 and nchan > 2: avimages, beamlist, freq_av, crms_av = self.windowaverage_cube(chan_images, rms_desired, chan_rms, c_wts, sbeam, freqin, nmax_to_avg=nmax_to_avg) total_flux, e_total_flux = self.fit_channels(img, avimages, crms_av, src, beamlist) src_total_flux = N.sum(total_flux, 1) # sum over all Gaussians in source to obtain total fluxes in each channel src_e_total_flux = N.sqrt(N.sum(N.power(e_total_flux, 2.0), 1)) src_mask, n_good_chan = self.mask_upper_limits(src_total_flux, src_e_total_flux, snr_desired) nchan = avimages.shape[0] if nchan == nchan_prev: break nchan_prev = nchan rms_desired *= 0.8 # Now fit source for spectral index. src_total_flux = src_total_flux.reshape((src_total_flux.shape[0],)) src_e_total_flux = src_e_total_flux.reshape((src_e_total_flux.shape[0],)) src_mask = src_mask.reshape((src_mask.shape[0],)) if img.opts.verbose_fitting: if img.opts.flagchan_snr: print 'Source #%i : averaged to %i channels, of which %i meet SNR criterion' % (src.source_id, len(src_total_flux), nchan) else: print 'Source #%i : averaged to %i channels, all of which will be used' % (src.source_id, len(src_total_flux)) npos = len(N.where(src_total_flux > 0.0)[0]) if isinstance(n_good_chan, int): n_good_chan = [n_good_chan] if (img.opts.flagchan_snr and n_good_chan[0] < 2) or npos < 2: src.spec_indx = N.NaN src.e_spec_indx = N.NaN src.spec_norm = N.NaN src.specin_flux = [N.NaN] src.specin_fluxE = [N.NaN] src.specin_freq = [N.NaN] src.specin_freq0 = N.NaN else: if img.opts.flagchan_snr: good_fluxes_ind = N.where(src_mask == False) else: good_fluxes_ind = range(len(freq_av)) fluxes_to_fit = src_total_flux[good_fluxes_ind] e_fluxes_to_fit = src_e_total_flux[good_fluxes_ind] freqs_to_fit = freq_av[good_fluxes_ind] # if len(freqs_to_fit.shape) == 2: # freqs_to_fit = freqs_to_fit.reshape((freqs_to_fit.shape[0],)) # if len(fluxes_to_fit.shape) == 2: # fluxes_to_fit = fluxes_to_fit.reshape((fluxes_to_fit.shape[0],)) # if len(e_fluxes_to_fit.shape) == 2: # e_fluxes_to_fit = e_fluxes_to_fit.reshape((e_fluxes_to_fit.shape[0],)) fit_res = self.fit_specindex(freqs_to_fit, fluxes_to_fit, e_fluxes_to_fit) src.spec_norm, src.spec_indx, src.e_spec_indx = fit_res src.specin_flux = fluxes_to_fit.tolist() src.specin_fluxE = e_fluxes_to_fit.tolist() src.specin_freq = freqs_to_fit.tolist() src.specin_freq0 = N.median(freqs_to_fit) else: src.spec_norm = src.gaussians[0].spec_norm src.spec_indx = src.gaussians[0].spec_indx src.e_spec_indx = src.gaussians[0].e_spec_indx src.specin_flux = src.gaussians[0].specin_flux src.specin_fluxE = src.gaussians[0].specin_fluxE src.specin_freq = src.gaussians[0].specin_freq src.specin_freq0 = src.gaussians[0].specin_freq0 if bar2.started: bar2.increment() img.completed_Ops.append('spectralindex') else: mylog.warning('Image has only one channel. Spectral index module disabled.') img.opts.spectralindex_do = False
def __call__(self, img): mylog = mylogger.logging.getLogger("PyBDSM." + img.log + "Polarisatn") if img.opts.polarisation_do: mylog.info('Extracting polarisation properties for all sources') pols = ['I', 'Q', 'U', 'V'] # Run gausfit and gual2srl on PI image to look for polarized sources # undetected in I fit_PI = img.opts.pi_fit n_new = 0 ch0_pi = N.sqrt(img.ch0_Q_arr**2 + img.ch0_U_arr**2) img.ch0_pi_arr = ch0_pi if fit_PI: from . import _run_op_list mylogger.userinfo(mylog, "\nChecking PI image for new sources") mask = img.mask_arr minsize = img.opts.minpix_isl # Set up image object for PI image. pi_chain, pi_opts = self.setpara_bdsm(img) pimg = Image(pi_opts) pimg.beam = img.beam pimg.pixel_beam = img.pixel_beam pimg.pixel_beamarea = img.pixel_beamarea pimg.log = 'PI.' pimg.pix2beam = img.pix2beam pimg.beam2pix = img.beam2pix pimg.pix2gaus = img.pix2gaus pimg.gaus2pix = img.gaus2pix pimg.pix2sky = img.pix2sky pimg.sky2pix = img.sky2pix pimg.pix2coord = img.pix2coord pimg.wcs_obj = img.wcs_obj pimg.mask_arr = mask pimg.masked = img.masked pimg.ch0_arr = ch0_pi pimg._pi = True success = _run_op_list(pimg, pi_chain) if not success: return img.pi_islands = pimg.islands img.pi_gaussians = pimg.gaussians img.pi_sources = pimg.sources # Now check for new sources in the PI image that are not # found in the Stokes I image. If any new sources are found, # adjust their IDs to follow after those found in I. new_isl = [] new_src = [] new_gaus = [] n_new_src = 0 isl_id = img.islands[-1].island_id src_id = img.sources[-1].source_id gaus_id = img.gaussians[-1].gaus_num for pi_isl in pimg.islands: new_sources = [] for pi_src in pi_isl.sources: if img.pyrank[int(img.sky2pix(pi_src.posn_sky_max)[0]), int(img.sky2pix(pi_src.posn_sky_max)[1] )] == -1: src_id += 1 pi_src._pi = True pi_src.island_id = isl_id pi_src.source_id = src_id pi_src.spec_indx = N.NaN pi_src.e_spec_indx = N.NaN pi_src.spec_norm = N.NaN pi_src.specin_flux = [N.NaN] pi_src.specin_fluxE = [N.NaN] pi_src.specin_freq = [N.NaN] pi_src.specin_freq0 = N.NaN new_sources.append(pi_src) new_src.append(pi_src) n_new_src += 1 for g in pi_src.gaussians: gaus_id += 1 new_gaus.append(g) g.gaus_num = gaus_id if len(new_sources) > 0: isl_id += 1 pi_isl.sources = new_sources pi_isl.island_id = isl_id pi_isl._pi = True new_isl.append(pi_isl) n_new = len(new_isl) mylogger.userinfo(mylog, "New sources found in PI image", '%i (%i total)' % (n_new, img.nsrc + n_new)) if n_new > 0: img.islands += new_isl img.sources += new_src img.gaussians += new_gaus img.nsrc += n_new_src bar = statusbar.StatusBar( 'Calculating polarisation properties .... : ', 0, img.nsrc) if img.opts.quiet == False: bar.start() for isl in img.islands: isl_bbox = isl.bbox ch0_I = img.ch0_arr[isl_bbox] ch0_Q = img.ch0_Q_arr[isl_bbox] ch0_U = img.ch0_U_arr[isl_bbox] ch0_V = img.ch0_V_arr[isl_bbox] ch0_images = [ch0_I, ch0_Q, ch0_U, ch0_V] for i, src in enumerate(isl.sources): # For each source, assume the morphology does not change # across the Stokes cube. This assumption allows us to fit # the Gaussians of each source to each Stokes image by # simply fitting only the overall normalizations of the # individual Gaussians. # # First, fit all source Gaussians to each Stokes image: x, y = N.mgrid[isl_bbox] gg = src.gaussians fitfix = N.ones(len(gg)) # fit only normalization srcmask = isl.mask_active total_flux = N.zeros( (4, len(fitfix)), dtype=N.float32 ) # array of fluxes: N_Stokes x N_Gaussians errors = N.zeros( (4, len(fitfix)), dtype=N.float32 ) # array of fluxes: N_Stokes x N_Gaussians for sind, image in enumerate(ch0_images): if (sind == 0 and hasattr(src, '_pi') ) or sind > 0: # Fit I only for PI sources p, ep = func.fit_mulgaus2d(image, gg, x, y, srcmask, fitfix) for ig in range(len(fitfix)): center_pix = (p[ig * 6 + 1], p[ig * 6 + 2]) bm_pix = N.array([ img.pixel_beam()[0], img.pixel_beam()[1], img.pixel_beam()[2] ]) total_flux[sind, ig] = p[ig * 6] * p[ ig * 6 + 3] * p[ig * 6 + 4] / (bm_pix[0] * bm_pix[1]) p = N.insert(p, N.arange(len(fitfix)) * 6 + 6, total_flux[sind]) if sind > 0: rms_img = img.__getattribute__('rms_' + pols[sind] + '_arr') else: rms_img = img.rms_arr if len(rms_img.shape) > 1: rms_isl = rms_img[isl.bbox].mean() else: rms_isl = rms_img errors[sind] = func.get_errors(img, p, rms_isl)[6] # Now, assign fluxes to each Gaussian. src_flux_I = 0.0 src_flux_Q = 0.0 src_flux_U = 0.0 src_flux_V = 0.0 src_flux_I_err_sq = 0.0 src_flux_Q_err_sq = 0.0 src_flux_U_err_sq = 0.0 src_flux_V_err_sq = 0.0 for ig, gaussian in enumerate(src.gaussians): flux_I = total_flux[0, ig] flux_I_err = abs(errors[0, ig]) flux_Q = total_flux[1, ig] flux_Q_err = abs(errors[1, ig]) flux_U = total_flux[2, ig] flux_U_err = abs(errors[2, ig]) flux_V = total_flux[3, ig] flux_V_err = abs(errors[3, ig]) if hasattr(src, '_pi'): gaussian.total_flux = flux_I gaussian.total_fluxE = flux_I_err gaussian.total_flux_Q = flux_Q gaussian.total_flux_U = flux_U gaussian.total_flux_V = flux_V gaussian.total_fluxE_Q = flux_Q_err gaussian.total_fluxE_U = flux_U_err gaussian.total_fluxE_V = flux_V_err if hasattr(src, '_pi'): src_flux_I += flux_I src_flux_I_err_sq += flux_I_err**2 src_flux_Q += flux_Q src_flux_U += flux_U src_flux_V += flux_V src_flux_Q_err_sq += flux_Q_err**2 src_flux_U_err_sq += flux_U_err**2 src_flux_V_err_sq += flux_V_err**2 # Calculate and store polarisation fractions and angle for each Gaussian in the island # For this we need the I flux, which we can just take from g.total_flux and src.total_flux flux_I = gaussian.total_flux flux_I_err = gaussian.total_fluxE stokes = [flux_I, flux_Q, flux_U, flux_V] stokes_err = [ flux_I_err, flux_Q_err, flux_U_err, flux_V_err ] lpol_frac, lpol_frac_loerr, lpol_frac_hierr = self.calc_lpol_fraction( stokes, stokes_err) # linear pol fraction lpol_ang, lpol_ang_err = self.calc_lpol_angle( stokes, stokes_err) # linear pol angle cpol_frac, cpol_frac_loerr, cpol_frac_hierr = self.calc_cpol_fraction( stokes, stokes_err) # circular pol fraction tpol_frac, tpol_frac_loerr, tpol_frac_hierr = self.calc_tpol_fraction( stokes, stokes_err) # total pol fraction gaussian.lpol_fraction = lpol_frac gaussian.lpol_fraction_loerr = lpol_frac_loerr gaussian.lpol_fraction_hierr = lpol_frac_hierr gaussian.cpol_fraction = cpol_frac gaussian.cpol_fraction_loerr = cpol_frac_loerr gaussian.cpol_fraction_hierr = cpol_frac_hierr gaussian.tpol_fraction = tpol_frac gaussian.tpol_fraction_loerr = tpol_frac_loerr gaussian.tpol_fraction_hierr = tpol_frac_hierr gaussian.lpol_angle = lpol_ang gaussian.lpol_angle_err = lpol_ang_err # Store fluxes for each source in the island if hasattr(src, '_pi'): src.total_flux = src_flux_I src.total_fluxE = N.sqrt(src_flux_I_err_sq) src.total_flux_Q = src_flux_Q src.total_flux_U = src_flux_U src.total_flux_V = src_flux_V src.total_fluxE_Q = N.sqrt(src_flux_Q_err_sq) src.total_fluxE_U = N.sqrt(src_flux_U_err_sq) src.total_fluxE_V = N.sqrt(src_flux_V_err_sq) # Calculate and store polarisation fractions and angle for each source in the island # For this we need the I flux, which we can just take from g.total_flux and src.total_flux src_flux_I = src.total_flux src_flux_I_err = src.total_fluxE stokes = [src_flux_I, src_flux_Q, src_flux_U, src_flux_V] stokes_err = [ src_flux_I_err, N.sqrt(src_flux_Q_err_sq), N.sqrt(src_flux_U_err_sq), N.sqrt(src_flux_V_err_sq) ] lpol_frac, lpol_frac_loerr, lpol_frac_hierr = self.calc_lpol_fraction( stokes, stokes_err) # linear pol fraction lpol_ang, lpol_ang_err = self.calc_lpol_angle( stokes, stokes_err) # linear pol angle cpol_frac, cpol_frac_loerr, cpol_frac_hierr = self.calc_cpol_fraction( stokes, stokes_err) # circular pol fraction tpol_frac, tpol_frac_loerr, tpol_frac_hierr = self.calc_tpol_fraction( stokes, stokes_err) # total pol fraction src.lpol_fraction = lpol_frac src.lpol_fraction_loerr = lpol_frac_loerr src.lpol_fraction_hierr = lpol_frac_hierr src.cpol_fraction = cpol_frac src.cpol_fraction_loerr = cpol_frac_loerr src.cpol_fraction_hierr = cpol_frac_hierr src.tpol_fraction = tpol_frac src.tpol_fraction_loerr = tpol_frac_loerr src.tpol_fraction_hierr = tpol_frac_hierr src.lpol_angle = lpol_ang src.lpol_angle_err = lpol_ang_err if bar.started: bar.increment() bar.stop() img.completed_Ops.append('polarisation')
def __call__(self, img): from time import time import functions as func import itertools mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Gausfit") if len(img.islands) == 0: img.gaussians = [] img.ngaus = 0 img.total_flux_gaus = 0.0 img.completed_Ops.append('gausfit') return img bar = statusbar.StatusBar('Fitting islands with Gaussians .......... : ', 0, img.nisl) opts = img.opts if opts.quiet == False and opts.verbose_fitting == False: bar.start() iter_ngmax = 10 min_maxsize = 50.0 maxsize = opts.splitisl_maxsize min_peak_size = 30.0 peak_size = opts.peak_maxsize if maxsize < min_maxsize: maxsize = min_maxsize opts.splitisl_maxsize = min_maxsize if peak_size < min_peak_size: peak_size = min_peak_size opts.peak_maxsize = min_peak_size # Set up multiproccessing. First create a simple copy of the Image # object that contains the minimal data needed. opts_dict = opts.to_dict() img_simple = Image(opts_dict) img_simple.pixel_beamarea = img.pixel_beamarea img_simple.pixel_beam = img.pixel_beam img_simple.thresh_pix = img.thresh_pix img_simple.minpix_isl = img.minpix_isl img_simple.clipped_mean = img.clipped_mean img_simple.beam2pix = img.beam2pix img_simple.beam = img.beam # Next, define the weights to use when distributing islands among cores. # The weight should scale with the processing time. At the moment # we use the island area, but other parameters may be better. weights = [] for isl in img.islands: weights.append(isl.size_active) # Now call the parallel mapping function. Returns a list of [gaul, fgaul] # for each island. gaus_list = mp.parallel_map(func.eval_func_tuple, itertools.izip(itertools.repeat(self.process_island), img.islands, itertools.repeat(img_simple), itertools.repeat(opts)), numcores=opts.ncores, bar=bar, weights=weights) for isl in img.islands: ### now convert gaussians into Gaussian objects and store idx = isl.island_id gaul = gaus_list[idx][0] fgaul = gaus_list[idx][1] dgaul = [] gaul = [Gaussian(img, par, idx, gidx) for (gidx, par) in enumerate(gaul)] if len(gaul) == 0: # No good Gaussians were fit. In this case, make a dummy # Gaussian located at the island center so # that the source may still be included in output catalogs. # These dummy Gaussians all have an ID of -1. They do not # appear in any of the source or island Gaussian lists except # the island dgaul list. if opts.src_ra_dec != None: # Center the dummy Gaussian on the user-specified source position posn_isl = (isl.shape[0]/2.0, isl.shape[1]/2.0) posn_img = (isl.shape[0]/2.0 + isl.origin[0], isl.shape[1]/2.0 + isl.origin[1]) par = [isl.image[posn_isl], posn_img[0], posn_img[1], 0.0, 0.0, 0.0] else: # Center the dummy Gaussian on the maximum pixel posn = N.unravel_index(N.argmax(isl.image*~isl.mask_active), isl.shape) + N.array(isl.origin) par = [isl.max_value, posn[0], posn[1], 0.0, 0.0, 0.0] dgaul = [Gaussian(img, par, idx, -1)] gidx = 0 fgaul= [Gaussian(img, par, idx, gidx + gidx2 + 1, flag) for (gidx2, (flag, par)) in enumerate(fgaul)] isl.gaul = gaul isl.fgaul= fgaul isl.dgaul = dgaul gaussian_list = [g for isl in img.islands for g in isl.gaul] img.gaussians = gaussian_list ### put in the serial number of the gaussians for the whole image n = 0 nn = 0 tot_flux = 0.0 if img.waveletimage: # store the wavelet scale for each Gaussian # (wavelet img's have a img.j attribute) j = img.j else: j = 0 for isl in img.islands: m = 0 for g in isl.gaul: n += 1; m += 1 g.gaus_num = n - 1 tot_flux += g.total_flux for dg in isl.dgaul: nn -= 1 dg.gaus_num = nn isl.ngaus = m img.ngaus = n img.total_flux_gaus = tot_flux mylogger.userinfo(mylog, "Total number of Gaussians fit to image", str(n)) if not img._pi and not img.waveletimage: mylogger.userinfo(mylog, "Total flux density in model", '%.3f Jy' % tot_flux) # Check if model flux is very different from sum of flux in image if img.ch0_sum_jy > 0 and not img._pi: if img.total_flux_gaus/img.ch0_sum_jy < 0.5 or \ img.total_flux_gaus/img.ch0_sum_jy > 2.0: mylog.warn('Total flux density in model is %0.2f times sum of pixels '\ 'in input image. Large residuals may remain.' % (img.total_flux_gaus/img.ch0_sum_jy,)) # Check if there are many Gaussians with deconvolved size of 0 in one # axis but not in the other. Don't bother to do this for wavelet images. fraction_1d = self.check_for_1d_gaussians(img) if fraction_1d > 0.5 and img.beam != None and img.waveletimage == False: mylog.warn('After deconvolution, more than 50% of Gaussians are '\ "1-D. Unless you're fitting an extended source, "\ "beam may be incorrect.") img.completed_Ops.append('gausfit') return img
def check_islands_for_overlap(img, wimg): """Checks for overlaps between img and wimg islands""" have_numexpr = True try: import numexpr as ne except: have_numexpr = False tot_flux = 0.0 bar = statusbar.StatusBar('Checking islands for overlap ............ : ', 0, len(wimg.islands)) # Make masks for regions that have islands wpp = wimg.pyrank + 1 # does not change, store for later wav_rankim_bool = wpp > 0 # boolean orig_rankim_bool = img.pyrank > -1 # Make "images" of island ids for overlaping regions orig_islands = wav_rankim_bool * (img.pyrank + 1) - 1 bar.start() for idx, wvisl in enumerate(wimg.islands): if len(wvisl.gaul) > 0: # Get unique island IDs. If an island overlaps with one # in the original ch0 image, merge them together. If not, # add the island as a new one. wav_islands = orig_rankim_bool[wvisl.bbox] * wpp[wvisl.bbox] - 1 wav_ids = N.unique(wav_islands) # saves conversion to set and back for wvg in wvisl.gaul: tot_flux += wvg.total_flux wvg.valid = True if idx in wav_ids: orig_idx = N.unique( orig_islands[wvisl.bbox][wav_islands == idx]) if len(orig_idx) == 1: merge_islands(img, img.islands[orig_idx[0]], wvisl) else: merge_islands(img, img.islands[orig_idx[0]], wvisl) for oidx in orig_idx[1:]: merge_islands(img, img.islands[orig_idx[0]], img.islands[oidx]) img.islands = [ x for x in img.islands if x.island_id not in orig_idx[1:] ] renumber_islands(img) # Now recalculate the overlap images, since the islands have changed ipp = img.pyrank + 1 if have_numexpr: orig_islands = ne.evaluate('wav_rankim_bool * ipp - 1') else: orig_islands = wav_rankim_bool * ipp - 1 else: isl_id = img.islands[-1].island_id + 1 new_isl = wvisl.copy(img.pixel_beamarea(), image=img.ch0_arr[wvisl.bbox], mean=img.mean_arr[wvisl.bbox], rms=img.rms_arr[wvisl.bbox]) new_isl.gaul = [] new_isl.dgaul = [] new_isl.island_id = isl_id img.islands.append(new_isl) copy_gaussians(img, new_isl, wvisl) bar.increment() bar.stop() return tot_flux
def __call__(self, img): if img.opts.psf_vary_do: mylog = mylogger.logging.getLogger("PyBDSM."+img.log+"Psf_Vary") mylogger.userinfo(mylog, '\nEstimating PSF variations') opts = img.opts dir = img.basedir + '/misc/' plot = False # debug figures image = img.ch0_arr try: from astropy.io import fits as pyfits old_pyfits = False except ImportError, err: from distutils.version import StrictVersion import pyfits if StrictVersion(pyfits.__version__) < StrictVersion('2.2'): old_pyfits = True else: old_pyfits = False if old_pyfits: mylog.warning('PyFITS version is too old: psf_vary module skipped') return if opts.psf_fwhm is not None: # User has specified a constant PSF to use, so skip PSF fitting/etc. psf_maj = opts.psf_fwhm[0] # FWHM in deg psf_min = opts.psf_fwhm[1] # FWHM in deg psf_pa = opts.psf_fwhm[2] # PA in deg mylogger.userinfo(mylog, 'Using constant PSF (major, minor, pos angle)', '(%.5e, %.5e, %s) degrees' % (psf_maj, psf_maj, round(psf_pa, 1))) else: # Use did not specify a constant PSF to use, so estimate it over = 2 generators = opts.psf_generators; nsig = opts.psf_nsig; kappa2 = opts.psf_kappa2 snrtop = opts.psf_snrtop; snrbot = opts.psf_snrbot; snrcutstack = opts.psf_snrcutstack gencode = opts.psf_gencode; primarygen = opts.psf_primarygen; itess_method = opts.psf_itess_method tess_sc = opts.psf_tess_sc; tess_fuzzy= opts.psf_tess_fuzzy bright_snr_cut = opts.psf_high_snr s_only = opts.psf_stype_only if opts.psf_snrcut < 5.0: mylogger.userinfo(mylog, "Value of psf_snrcut too low; increasing to 5") snrcut = 5.0 else: snrcut = opts.psf_snrcut img.psf_snrcut = snrcut if opts.psf_high_snr is not None: if opts.psf_high_snr < 10.0: mylogger.userinfo(mylog, "Value of psf_high_snr too low; increasing to 10") high_snrcut = 10.0 else: high_snrcut = opts.psf_high_snr else: high_snrcut = opts.psf_high_snr img.psf_high_snr = high_snrcut wtfns=['unity', 'roundness', 'log10', 'sqrtlog10'] if 0 <= itess_method < 4: tess_method=wtfns[itess_method] else: tess_method='unity' ### now put all relevant gaussian parameters into a list ngaus = img.ngaus nsrc = img.nsrc num = N.zeros(nsrc, dtype=N.int32) peak = N.zeros(nsrc) xc = N.zeros(nsrc) yc = N.zeros(nsrc) bmaj = N.zeros(nsrc) bmin = N.zeros(nsrc) bpa = N.zeros(nsrc) code = N.array(['']*nsrc); rms = N.zeros(nsrc) src_id_list = [] for i, src in enumerate(img.sources): src_max = 0.0 for gmax in src.gaussians: # Take only brightest Gaussian per source if gmax.peak_flux > src_max: src_max = gmax.peak_flux g = gmax num[i] = i peak[i] = g.peak_flux xc[i] = g.centre_pix[0] yc[i] = g.centre_pix[1] bmaj[i] = g.size_pix[0] bmin[i] = g.size_pix[1] bpa[i] = g.size_pix[2] code[i] = img.sources[g.source_id].code rms[i] = img.islands[g.island_id].rms gauls = (num, peak, xc, yc, bmaj, bmin, bpa, code, rms) tr_gauls = self.trans_gaul(gauls) # takes gaussians with code=S and snr > snrcut. if s_only: tr = [n for n in tr_gauls if n[1]/n[8]>snrcut and n[7] == 'S'] else: tr = [n for n in tr_gauls if n[1]/n[8]>snrcut] g_gauls = self.trans_gaul(tr) # computes statistics of fitted sizes. Same as psfvary_fullstat.f in fBDSM. bmaj_a, bmaj_r, bmaj_ca, bmaj_cr, ni = _cbdsm.bstat(bmaj, None, nsig) bmin_a, bmin_r, bmin_ca, bmin_cr, ni = _cbdsm.bstat(bmin, None, nsig) bpa_a, bpa_r, bpa_ca, bpa_cr, ni = _cbdsm.bstat(bpa, None, nsig) # get subset of sources deemed to be unresolved. Same as size_ksclip_wenss.f in fBDSM. flag_unresolved = self.get_unresolved(g_gauls, img.beam, nsig, kappa2, over, img.psf_high_snr, plot) if len(flag_unresolved) == 0: mylog.warning('Insufficient number of sources to determine PSF variation.\nTry changing the PSF options or specify a (constant) PSF with the "psf_fwhm" option') return # see how much the SNR-weighted sizes of unresolved sources differ from the synthesized beam. wtsize_beam_snr = self.av_psf(g_gauls, img.beam, flag_unresolved) # filter out resolved sources tr_gaul = self.trans_gaul(g_gauls) tr = [n for i, n in enumerate(tr_gaul) if flag_unresolved[i]] g_gauls = self.trans_gaul(tr) mylogger.userinfo(mylog, 'Number of unresolved sources', str(len(g_gauls[0]))) # get a list of voronoi generators. vorogenS has values (and not None) if generators='field'. vorogenP, vorogenS = self.get_voronoi_generators(g_gauls, generators, gencode, snrcut, snrtop, snrbot, snrcutstack) mylogger.userinfo(mylog, 'Number of generators for PSF variation', str(len(vorogenP[0]))) if len(vorogenP[0]) < 3: mylog.warning('Insufficient number of generators') return mylogger.userinfo(mylog, 'Tesselating image') # group generators into tiles tile_prop = self.edit_vorogenlist(vorogenP, frac=0.9) # tesselate the image volrank, vorowts = self.tesselate(vorogenP, vorogenS, tile_prop, tess_method, tess_sc, tess_fuzzy, \ generators, gencode, image.shape) if opts.output_all: func.write_image_to_file(img.use_io, img.imagename + '.volrank.fits', volrank, img, dir) tile_list, tile_coord, tile_snr = tile_prop ntile = len(tile_list) bar = statusbar.StatusBar('Determining PSF variation ............... : ', 0, ntile) mylogger.userinfo(mylog, 'Number of tiles for PSF variation', str(ntile)) # For each tile, calculate the weighted averaged psf image. Also for all the sources in the image. cdelt = list(img.wcs_obj.acdelt[0:2]) factor=3. psfimages, psfcoords, totpsfimage, psfratio, psfratio_aper = self.psf_in_tile(image, img.beam, g_gauls, \ cdelt, factor, snrcutstack, volrank, tile_prop, plot, img) npsf = len(psfimages) if opts.psf_use_shap: if opts.psf_fwhm is None: # use totpsfimage to get beta, centre and nmax for shapelet decomposition. Use nmax=5 or 6 mask=N.zeros(totpsfimage.shape, dtype=bool) (m1, m2, m3)=func.moment(totpsfimage, mask) betainit=sqrt(m3[0]*m3[1])*2.0 * 1.4 tshape = totpsfimage.shape cen = N.array(N.unravel_index(N.argmax(totpsfimage), tshape))+[1,1] cen = tuple(cen) nmax = 12 basis = 'cartesian' betarange = [0.5,sqrt(betainit*max(tshape))] beta, error = sh.shape_varybeta(totpsfimage, mask, basis, betainit, cen, nmax, betarange, plot) if error == 1: print ' Unable to find minimum in beta' # decompose all the psf images using the beta from above nmax=12; psf_cf=[] for i in range(npsf): psfim = psfimages[i] cf = sh.decompose_shapelets(psfim, mask, basis, beta, cen, nmax, mode='') psf_cf.append(cf) if img.opts.quiet == False: bar.increment() bar.stop() # transpose the psf image list xt, yt = N.transpose(tile_coord) tr_psf_cf = N.transpose(N.array(psf_cf)) # interpolate the coefficients across the image. Ok, interpolate in scipy for # irregular grids is crap. doesnt even pass through some of the points. # for now, fit polynomial. compress = 100.0 x, y = N.transpose(psfcoords) if len(x) < 3: mylog.warning('Insufficient number of tiles to do interpolation of PSF variation') return psf_coeff_interp, xgrid, ygrid = self.interp_shapcoefs(nmax, tr_psf_cf, psfcoords, image.shape, \ compress, plot) psfshape = psfimages[0].shape skip = 5 aa = self.create_psf_grid(psf_coeff_interp, image.shape, xgrid, ygrid, skip, nmax, psfshape, \ basis, beta, cen, totpsfimage, plot) img.psf_images = aa else: if opts.psf_fwhm is None: if ntile < 4: mylog.warning('Insufficient number of tiles to do interpolation of PSF variation') return else: # Fit stacked PSFs with Gaussians and measure aperture fluxes bm_pix = N.array([img.pixel_beam()[0]*fwsig, img.pixel_beam()[1]*fwsig, img.pixel_beam()[2]]) psf_maj = N.zeros(npsf) psf_min = N.zeros(npsf) psf_pa = N.zeros(npsf) if img.opts.quiet == False: bar.start() for i in range(ntile): psfim = psfimages[i] mask = N.zeros(psfim.shape, dtype=bool) x_ax, y_ax = N.indices(psfim.shape) maxv = N.max(psfim) p_ini = [maxv, (psfim.shape[0]-1)/2.0*1.1, (psfim.shape[1]-1)/2.0*1.1, bm_pix[0]/fwsig*1.3, bm_pix[1]/fwsig*1.1, bm_pix[2]*2] para, ierr = func.fit_gaus2d(psfim, p_ini, x_ax, y_ax, mask) ### first extent is major if para[3] < para[4]: para[3:5] = para[4:2:-1] para[5] += 90 ### clip position angle para[5] = divmod(para[5], 180)[1] psf_maj[i] = para[3] psf_min[i] = para[4] posang = para[5] while posang >= 180.0: posang -= 180.0 psf_pa[i] = posang if img.opts.quiet == False: bar.increment() bar.stop() # Interpolate Gaussian parameters if img.aperture is None: psf_maps = [psf_maj, psf_min, psf_pa, psfratio] else: psf_maps = [psf_maj, psf_min, psf_pa, psfratio, psfratio_aper] nimgs = len(psf_maps) bar = statusbar.StatusBar('Interpolating PSF images ................ : ', 0, nimgs) if img.opts.quiet == False: bar.start() map_list = mp.parallel_map(func.eval_func_tuple, itertools.izip(itertools.repeat(self.interp_prop), psf_maps, itertools.repeat(psfcoords), itertools.repeat(image.shape)), numcores=opts.ncores, bar=bar) if img.aperture is None: psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int = map_list else: psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int, psf_ratio_aper_int = map_list # Smooth if desired if img.opts.psf_smooth is not None: sm_scale = img.opts.psf_smooth / img.pix2beam([1.0, 1.0, 0.0])[0] / 3600.0 # pixels if img.opts.aperture is None: psf_maps = [psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int] else: psf_maps = [psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int, psf_ratio_aper_int] nimgs = len(psf_maps) bar = statusbar.StatusBar('Smoothing PSF images .................... : ', 0, nimgs) if img.opts.quiet == False: bar.start() map_list = mp.parallel_map(func.eval_func_tuple, itertools.izip(itertools.repeat(self.blur_image), psf_maps, itertools.repeat(sm_scale)), numcores=opts.ncores, bar=bar) if img.aperture is None: psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int = map_list else: psf_maj_int, psf_min_int, psf_pa_int, psf_ratio_int, psf_ratio_aper_int = map_list # Make sure all smoothed, interpolated images are ndarrays psf_maj_int = N.array(psf_maj_int) psf_min_int = N.array(psf_min_int) psf_pa_int = N.array(psf_pa_int) psf_ratio_int = N.array(psf_ratio_int) if img.aperture is None: psf_ratio_aper_int = N.zeros(psf_maj_int.shape, dtype=N.float32) else: psf_ratio_aper_int = N.array(psf_ratio_aper_int, dtype=N.float32) # Blank with NaNs if needed mask = img.mask_arr if isinstance(mask, N.ndarray): pix_masked = N.where(mask == True) psf_maj_int[pix_masked] = N.nan psf_min_int[pix_masked] = N.nan psf_pa_int[pix_masked] = N.nan psf_ratio_int[pix_masked] = N.nan psf_ratio_aper_int[pix_masked] = N.nan # Store interpolated images. The major and minor axis images are # the sigma in units of arcsec, the PA image in units of degrees east of # north, the ratio images in units of 1/beam. img.psf_vary_maj_arr = psf_maj_int * img.pix2beam([1.0, 1.0, 0.0])[0] * 3600.0 # sigma in arcsec img.psf_vary_min_arr = psf_min_int * img.pix2beam([1.0, 1.0, 0.0])[0] * 3600.0 # sigma in arcsec img.psf_vary_pa_arr = psf_pa_int img.psf_vary_ratio_arr = psf_ratio_int # in 1/beam img.psf_vary_ratio_aper_arr = psf_ratio_aper_int # in 1/beam if opts.output_all: func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_maj.fits', img.psf_vary_maj_arr*fwsig, img, dir) func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_min.fits', img.psf_vary_min_arr*fwsig, img, dir) func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_pa.fits', img.psf_vary_pa_arr, img, dir) func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_ratio.fits', img.psf_vary_ratio_arr, img, dir) func.write_image_to_file(img.use_io, img.imagename + '.psf_vary_ratio_aper.fits', img.psf_vary_ratio_aper_arr, img, dir) # Loop through source and Gaussian lists and deconvolve the sizes using appropriate beam bar2 = statusbar.StatusBar('Correcting deconvolved source sizes ..... : ', 0, img.nsrc) if img.opts.quiet == False: bar2.start() for src in img.sources: src_pos = img.sky2pix(src.posn_sky_centroid) src_pos_int = (int(src_pos[0]), int(src_pos[1])) gaus_c = img.gaus2pix(src.size_sky, src.posn_sky_centroid) if opts.psf_fwhm is None: gaus_bm = [psf_maj_int[src_pos_int]*fwsig, psf_min_int[src_pos_int]*fwsig, psf_pa_int[src_pos_int]] else: # Use user-specified constant PSF instead gaus_bm = img.beam2pix(opts.psf_fwhm) gaus_dc, err = func.deconv2(gaus_bm, gaus_c) src.deconv_size_sky = img.pix2gaus(gaus_dc, src_pos) src.deconv_size_skyE = [0.0, 0.0, 0.0] for g in src.gaussians: gaus_c = img.gaus2pix(g.size_sky, src.posn_sky_centroid) gaus_dc, err = func.deconv2(gaus_bm, gaus_c) g.deconv_size_sky = img.pix2gaus(gaus_dc, g.centre_pix) g.deconv_size_skyE = [0.0, 0.0, 0.0] if img.opts.quiet == False: bar2.spin() if img.opts.quiet == False: bar2.increment() bar2.stop() img.completed_Ops.append('psf_vary')