def do_sourcefinding(imagename, si=True): # get beam info manually. SKA image seems to cause PyBDSF issues finding this info. f = fits.open(imagename) beam_maj = f[0].header['BMAJ'] beam_min = f[0].header['BMIN'] #beam_pa = f[0].header['BPA'] # not in SKA fits header, but we know it's circular beam_pa = 0 f.close() # using some sensible and thorough hyper-parameters. PSF_vary and adaptive_rms_box is more computationally intensive, but needed. if si == True: img = bdsf.process_image(imagename, adaptive_rms_box=True, spectralindex_do=True, advanced_opts=True,\ atrous_do=False, psf_vary_do=True, psf_snrcut=5.0, psf_snrcutstack=10.0,\ output_opts=True, output_all=True, opdir_overwrite='append', beam=(beam_maj, beam_min, beam_pa),\ blank_limit=None, thresh='hard', thresh_isl=5.0, thresh_pix=7.0, psf_snrtop=0.30,\ collapse_mode='single') # use 560 Mhz image as ch0 # save the img object as a pickle file, so we can do interactive checks after pybdsf has run save_obj(img, 'pybdsf_processimage_' + imagename[:-5]) if si == False: img = bdsf.process_image(imagename, adaptive_rms_box=True, advanced_opts=True,\ atrous_do=False, psf_vary_do=True, psf_snrcut=5.0, psf_snrcutstack=10.0,\ output_opts=True, output_all=True, opdir_overwrite='append', beam=(beam_maj, beam_min, beam_pa),\ blank_limit=None, thresh='hard', thresh_isl=5.0, thresh_pix=7.0, psf_snrtop=0.30) # save the img object as a pickle file, so we can do interactive checks after pybdsf has run save_obj(img, 'pybdsf_processimage_noSI_' + imagename[:-5])
def find_sources(self): """Run PyBDSF ``process_image()`` task using object attributes as parameter inputs. Returns ``None`` if PyBDSF fails. Wrapped in a timeout function so processing is killed if taking longer than 5 minutes. """ sf_logger.info('Extracting sources...') start = datetime.now() opts = self.get_attr() with warnings.catch_warnings(): warnings.filterwarnings('ignore', r'invalid value') try: out = bdsf.process_image(opts) except: out = None try: sf_logger.info(' -- found {} sources in {:.2f} seconds'.format( out.nsrc, (datetime.now() - start).total_seconds())) except AttributeError: try: sf_logger.info(' -- found {} islands in {:.2f} seconds'.format( out.nisl, (datetime.now() - start).total_seconds())) except AttributeError: sf_logger.info(' -- PyBDSF failed to process image.') return out
def RunBDSM(self, filename, rmsmean_map_filename=None, WriteNoise=True, Ratio=None): CatName = filename[:-5] + ".pybdsm.gaul.fits" if os.path.isfile(CatName): print >> log, ModColor.Str("File %s exists" % CatName, col="green") print >> log, ModColor.Str(" Skipping...", col="green") return rmsmean_map_filename = [] if self.rmsmean_map: l = self.rmsmean_map l = l.replace("[", "") l = l.replace("]", "") l = l.split(",") rmsmean_map_filename = l img = bdsf.process_image(filename, thresh_isl=self.bdsm_thresh_isl, thresh_pix=self.bdsm_thresh_pix, rms_box=self.bdsm_rms_box, rmsmean_map_filename=rmsmean_map_filename) img.write_catalog(catalog_type="srl", clobber=True, format="fits") img.write_catalog(catalog_type="gaul", clobber=True, format="fits") if WriteNoise: img.export_image(img_type="rms", clobber=True) img.export_image(img_type="mean", clobber=True)
def make_mask(image_name, mask_name=None, threshisl=5, atrous_do=False, rmsbox=(100,30), mask_combine=None): import sys, os import numpy as np from astropy.io import fits as pyfits import bdsf # wavelets are required to fit gaussians if atrous_do: stop_at = None else: stop_at = 'isl' # DO THE SOURCE DETECTION img = bdsf.process_image(image_name, rms_box=rmsbox, frequency=54e6, \ thresh_isl=float(threshisl), thresh_pix=float(threshisl*3./5), atrous_do=atrous_do, atrous_jmax=3, \ adaptive_rms_box=True, adaptive_thresh=100, rms_box_bright=(30,10), stop_at=stop_at, quiet=True, debug=False) # WRITE THE MASK FITS if mask_name == None: mask_name = image_name+'.newmask' if os.path.exists(mask_name): os.system('rm -r ' + mask_name) img.export_image(img_type='island_mask', img_format='fits', outfile=mask_name) del img # do an pixel-by-pixel "OR" operation with a given mask if not mask_combine is None: print("Doing a pix-by-pix OR with %s." % mask_combine) with pyfits.open(mask_combine) as fits: data_comb = fits[0].data with pyfits.open(mask_name) as fits: data = fits[0].data assert data.shape() == data_comb.shape() data[(data_comb == 1.)] = 1. fits[0].data = data fits.writeto(mask_name, overwrite=True) return mask_name
def find_sources(self): """ Find sources in smoothed & pb-corrected image Write them to a temp file in working directory """ #dont' do source finding if output already exists if not os.path.exists(self.bdsf_output) and not self.skipcheck: try: #use all defaults ot match what ALexander does bdsf.process_image(self.pbsmfits).write_catalog( outfile=self.bdsf_output, format='ascii', clobber=True) except Exception as e: self.status = False print( ("Source finding failed for " "beam {0} of taskid {1}").format(self.beam, self.taskid)) print(e)
def extract_tier1(input_image, options=options_tier1): """ Extract the sources from a fits image using PyBDSF. By default it applies the options used for Tier1. """ img = bdsf.process_image(input_image, **options) if not options["output_all"]: img.write_catalog(format='bbs', clobber=True) img.write_catalog(format='fits', catalog_type='gaul', clobber=True) img.write_catalog(format='fits', catalog_type='srl', clobber=True)
def find_sources(self, sf="aegean", options={}): outfile = self.imagename.replace(".fits", "_comp.csv") if sf == "aegean": command = "aegean " for opt in sorted(options): if opt == "table": continue command += "--{} {} ".format(opt, options[opt]) command += "--table {} ".format( self.imagename.replace(".fits", ".csv")) command += self.image + " " command += "> " + self.imagename.replace(".fits", "_aegean.log") self.logger.debug("Aegean command: {}".format(command)) subprocess.call(command, shell=True) elif sf == "pybdsf": self.logger.info("Running source finding with PyBDSF...") kwargs = options # procimg_kwargs.update(options) pybdsf_img = bdsf.process_image(self.image, **kwargs) pybdsf_img.write_catalog(outfile=outfile + ".original", catalog_type="gaul", format="csv") utils.pybdsf2aegean(outfile + ".original", outfile) self.logger.info( "PyBDSF output converted to aegean format and placed in {}.". format(outfile)) elif sf == "selavy": self.logger.info("Running source finding with Selavy...") selavy_parset_name = self.imagename.replace(".fits", ".selavy.in") #Add the image to the parset selavy_outputname = self.imagename.replace(".fits", ".txt") options["Selavy.image"] = self.image options["Selavy.resultsFile"] = selavy_outputname self._write_selavy_parset(self, selavy_parset_name, options) self.logger.info( "Written Selavy parset file {}.".format(selavy_parset_name)) if "Selavy.nsubx" in options: nsubx = int(options["Selavy.nsubx"]) if "Selavy.nsuby" in options: nsuby = int(options["Selavy.nsuby"]) num_procs = nsubx * nsuby + 1 command = "mpirun -n {} selavy -c {} > ".format( num_procs, selavy_parset_name) + self.imagename.replace( ".fits", "_selavy.log") subprocess.call(command, shell=True) utils.selavy2aegean( selavy_outputname.replace(".txt", ".components.txt"), outfile) self.logger.info( "Selavy output converted to aegean format and placed in {}.". format(outfile))
def make_mask(image_name, mask_name=None, threshisl=5, atrous_do=False, rmsbox=(100, 10), adaptive_thresh=50, mask_combine=None, write_srl=False): import sys, os import numpy as np from astropy.io import fits as pyfits import bdsf # wavelets are required to fit gaussians if atrous_do or write_srl: stop_at = None else: stop_at = 'isl' # DO THE SOURCE DETECTION img = bdsf.process_image(image_name, rms_box=rmsbox, frequency=54e6, \ thresh_isl=float(threshisl), thresh_pix=float(threshisl*3/5.), rms_map=True, mean_map='zero', atrous_do=atrous_do, atrous_jmax=4, \ adaptive_rms_box=True, adaptive_thresh=adaptive_thresh, rms_box_bright=(30,5), \ flagging_opts=True, flag_maxsize_fwhm=0.5, stop_at=stop_at, quiet=True, debug=False) # WRITE THE MASK FITS if mask_name == None: mask_name = image_name + '.newmask' if os.path.exists(mask_name): os.system('rm -r ' + mask_name) img.export_image(img_type='island_mask', img_format='fits', outfile=mask_name, clobber=True) # WRITE CATALOGUE if write_srl: img.write_catalog(format='bbs', catalog_type='gaul', outfile=mask_name.replace('fits', 'skymodel'), clobber=True) del img # do an pixel-by-pixel "OR" operation with a given mask if not mask_combine is None: print("Doing a pix-by-pix OR with %s." % mask_combine) with pyfits.open(mask_combine) as fits: data_comb = fits[0].data with pyfits.open(mask_name) as fits: data = fits[0].data assert data.shape() == data_comb.shape() data[(data_comb == 1.)] = 1. fits[0].data = data fits.writeto(mask_name, overwrite=True) return mask_name
def do_sourcefinding(imagename): # get beam info manually. SKA image seems to cause PyBDSF issues finding this info. f = fits.open(imagename) beam_maj = f[0].header['BMAJ'] beam_min = f[0].header['BMIN'] #beam_pa = f[0].header['BPA'] # not in SKA fits header, but we know it's circular beam_pa = 0 f.close() # Run sourcefinding using some sensible hyper-parameters. PSF_vary and adaptive_rms_box is more computationally intensive, off for now img = bdsf.process_image(imagename, adaptive_rms_box=False, advanced_opts=True,\ atrous_do=False, psf_vary_do=False, psf_snrcut=5.0, psf_snrcutstack=10.0,\ output_opts=True, output_all=True, opdir_overwrite='append', beam=(beam_maj, beam_min, beam_pa),\ blank_limit=None, thresh='hard', thresh_isl=5.0, thresh_pix=7.0, psf_snrtop=0.30) #
def create_mask(self, image, mask, threshold, theoretical_noise, beampars=None, rms_map=None): """ Creates a mask from an image using pybdsf image (string): Input image to use in MIRIAD format mask (string): Output mask image in MIRIAD format threshold (float): Threshold in Jy to use theoretical_noise (float): Theoretical noise for calculating the adaptive threshold parameter inside pybdsf """ convim.mirtofits(image, image + '.fits') bdsf_threshold = threshold / theoretical_noise if beampars: # bdsf.process_image(image + '.fits', stop_at='isl', thresh_isl=bdsf_threshold, beam=beampars, adaptive_rms_box=True, rms_map=rms_map).export_image(outfile=mask + '.fits', img_format='fits', img_type='island_mask', pad_image=True) bdsf.process_image(image + '.fits', stop_at='isl', thresh_isl=bdsf_threshold, beam=beampars, adaptive_rms_box=True, rms_map=False, rms_value=theoretical_noise).export_image(outfile=mask + '.fits', img_format='fits', img_type='island_mask', pad_image=True) else: bdsf.process_image(image + '.fits', stop_at='isl', thresh_isl=bdsf_threshold, adaptive_rms_box=True, rms_map=False, rms_value=theoretical_noise).export_image(outfile=mask + '.fits', img_format='fits', img_type='island_mask', pad_image=True) if os.path.isfile(mask + '.fits'): # Add a random number to the masks to make it viewable in kvis fitsmask = pyfits.open(mask + '.fits') fitsmask_data = fitsmask[0].data fitsmask_hdr = fitsmask[0].header rand_array = np.random.rand(int(fitsmask_hdr['NAXIS1']), int(fitsmask_hdr['NAXIS2'])) fitsmask[0].data = np.multiply(rand_array, fitsmask_data) fitsmask.writeto(mask + '.fits', clobber=True) # Convert mask to MIRIAD and generate a usable one for MIRIAD convim.fitstomir(mask + '.fits', mask + '_pybdsf') maths = lib.miriad('maths') maths.out = mask maths.exp = '"<' + mask + '_pybdsf>"' maths.mask = '"<' + mask + '_pybdsf>.gt.0' + '"' maths.go() managefiles.director(self, 'rm', image + '.fits.pybdsf.log') managefiles.director(self, 'rm', image + '.fits') managefiles.director(self, 'rm', mask + '.fits') managefiles.director(self, 'rm', mask + '_pybdsf') else: pass
def calc_shift(self, ref_cat, separation=15): """ Find a shift cross-matching source extracted from the image and a given catalog separation in arcsec """ import bdsf from astropy.coordinates import match_coordinates_sky from astropy.coordinates import SkyCoord import astropy.units as u img_cat = self.imagefile + '.cat' if not os.path.exists(img_cat): bdsf_img = bdsf.process_image(self.imagefile, rms_box=(100,30), \ thresh_pix=5, thresh_isl=3, atrous_do=False, \ adaptive_rms_box=True, adaptive_thresh=100, rms_box_bright=(30,10)) bdsf_img.write_catalog(outfile=img_cat, catalog_type='srl', format='fits', clobber=True) # TODO: add iterations to remove off sources until convergence # read catlogue ref_t = Table.read(ref_cat) img_t = Table.read(img_cat) # cross match idx_match, sep, _ = match_coordinates_sky(SkyCoord(ref_t['RA'], ref_t['DEC']),\ SkyCoord(img_t['RA'], img_t['DEC'])) idx_match_img = idx_match[sep < separation * u.arcsec] idx_match_ref = np.arange(0, len(ref_t))[sep < separation * u.arcsec] # find & apply shift if len(idx_match) == 0: logging.warning('No match found in TGSS.') return dra = ref_t['RA'][idx_match_ref] - img_t['RA'][idx_match_img] dra[dra > 180] -= 360 dra[dra < -180] += 360 ddec = ref_t['DEC'][idx_match_ref] - img_t['DEC'][idx_match_img] self.apply_shift(np.mean(dra), np.mean(ddec)) # clean up if not args.save: os.system('rm ' + img_cat)
def make_catalogue(self): """ Create catalogue for this image """ import bdsf from astropy.table import Table img_cat = self.imagefile+'.cat' if not os.path.exists(img_cat): bdsf_img = bdsf.process_image(self.imagefile, rms_box=(100,30), \ thresh_pix=5, thresh_isl=3, atrous_do=False, \ adaptive_rms_box=True, adaptive_thresh=100, rms_box_bright=(30,10), quiet=True) bdsf_img.write_catalog(outfile=img_cat, catalog_type='srl', format='fits', clobber=True) else: logging.warning('%s already exists, using it.' % img_cat) self.cat = Table.read(img_cat) logging.debug('%s: Number of sources detected: %i' % (self.imagefile, len(self.cat)) )
def make_catalogue(self): """ Create catalogue for this image. """ import bdsf from astropy.table import Table img_cat = self.imagefile + '.cat' if not os.path.exists(img_cat): bdsf_img = bdsf.process_image(self.imagefile, rms_box=(100,30), \ thresh_pix=5, thresh_isl=3, atrous_do=False, \ adaptive_rms_box=True, adaptive_thresh=100, rms_box_bright=(30,10), quiet=True) bdsf_img.write_catalog(outfile=img_cat, catalog_type='srl', format='fits', clobber=True) bdsf_img.write_catalog(outfile=img_cat.replace( '.cat', '.skymodel'), catalog_type='gaul', format='bbs', bbs_patches='source', clobber=True, srcroot='src') else: logging.warning('%s already exists, using it.' % img_cat) cat = Table.read(img_cat) # remove extended sources extended_src = (cat['Peak_flux'] / cat['Total_flux']) < 0.1 # ~extended source extended_src[cat['S_Code'] == 'M'] = True # multiple-gaussian source extended_src[cat['S_Code'] == 'C'] = True # one gaussian + other sources island # remove same sources from skymodel cat_lsm = lsm.load(img_cat.replace('.cat', '.skymodel')) for srcid in cat[extended_src]['Source_id']: cat_lsm.remove(f'Patch == src_patch_s{srcid}') cat.remove_rows(np.argwhere(extended_src)) self.cat = cat self.cat_lsm = cat_lsm logging.debug( '%s: Number of sources detected: %i; removed %i extended sources.' % (self.imagefile, len(self.cat), sum(extended_src)))
def makeLSM(infits): #,mybeam,myfreq): img = bdsf.process_image(infits, thresh_pix=9.0, thresh_isl=8.0) #,beam=mybeam,frequency=myfreq) foundsrcs = img.write_catalog(format='ascii', catalog_type='gaul', clobber=True, incl_empty=True) gaul = infits.replace('.fits', '.pybdsm.gaul') oplsm = gaul.replace('gaul', 'lsm.html') if foundsrcs: tiggerConvert(gaul) if cleanup: os.system('rm ' + gaul) os.system('rm ' + gaul.replace('.pybdsm.gaul', '.fits.pybdsm.log')) gi('Wrote LSM: ' + oplsm) return (oplsm, True) else: ri('No sources found in ' + str(infits)) ri('Could be corrupt image, could be trouble source entering null') return (oplsm, False)
def _process_image(self, fitsfile, thresh_isl, thresh_pix): """ Executes PyBDSF on the fitsfile Parameters ---------- fitsfile : string Name of input fitsfile thresh_isl : float Threshold for the island boundary in number of sigma above the mean. Determines extent of island used for fitting. thresh_pix : float Source detection threshold: threshold for the island peak in number of sigma above the mean. """ bdsf_dict = { 'rms_map': None, 'thresh': 'hard', 'thresh_isl': thresh_isl, 'thresh_pix': thresh_pix } return bdsf.process_image(fitsfile, **bdsf_dict)
def run_pybdsf(fitsname, detectimage, outcat='skymodel_1asec_lbregion_pybdsf'): ''' Run PyBDSF on an image, using standard SKSP settings. Two catalogues are written when finished: - A CSV catalogue with all columns present. - A BBS formatted catalogue, suitable for e.g. DPPP. Args: fitsname (str): path to the image from which to extract the fluxes. detectimage (str): path to the image on which to run source detection. Returns: None ''' # Pull the reference frequency from the header. f = fits.open(fitsname) restfrq = f[0].header['CRVAL3'] # Run PyBDSF with standard SKSP settings. res = bdsf.process_image(fitsname, detection_image=detectimage, thresh_isl=4.0, thresh_pix=5.0, rms_box=(150, 15), rms_map=True, mean_map='zero', ini_method='intensity', adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(60, 15), group_by_isl=False, group_tol=10.0, output_opts=True, output_all=True, atrous_do=True, atrous_jmax=4, flagging_opts=True, flag_maxsize_fwhm=0.5, advanced_opts=True, blank_limit=None, frequency=restfrq) # Write out a catalog. res.write_catalog(outfile=outcat + '.csv', bbs_patches='source', catalog_type='gaul', format='csv') res.write_catalog(outfile=outcat + '.bbs', bbs_patches='source', catalog_type='gaul', format='bbs')
def calc_shift(self, ref_cat, separation=15): """ Find a shift cross-matching source extracted from the image and a given catalog separation in arcsec """ import bdsf from astropy.coordinates import match_coordinates_sky from astropy.coordinates import SkyCoord import astropy.units as u img_cat = self.imagefile+'.cat' if not os.path.exists(img_cat): bdsf_img = bdsf.process_image(self.imagefile, rms_box=(100,30), \ thresh_pix=5, thresh_isl=3, atrous_do=False, \ adaptive_rms_box=True, adaptive_thresh=100, rms_box_bright=(30,10), quiet=True) bdsf_img.write_catalog(outfile=img_cat, catalog_type='srl', format='fits', clobber=True) # read catlogue ref_t = Table.read(ref_cat) img_t = Table.read(img_cat) # cross match idx_match, sep, _ = match_coordinates_sky(SkyCoord(ref_t['RA'], ref_t['DEC']),\ SkyCoord(img_t['RA'], img_t['DEC'])) idx_match_img = idx_match[sep<separation*u.arcsec] idx_match_ref = np.arange(0,len(ref_t))[sep<separation*u.arcsec] # find & apply shift if len(idx_match) == 0: logging.warning('No match found in TGSS.') return dra = ref_t['RA'][idx_match_ref] - img_t['RA'][idx_match_img] dra[ dra>180 ] -= 360 dra[ dra<-180 ] += 360 ddec = ref_t['DEC'][idx_match_ref] - img_t['DEC'][idx_match_img] self.apply_shift(np.mean(dra), np.mean(ddec)) # clean up if not args.save: os.system('rm '+img_cat)
def sfind_image(catprefix,pbimage,nonpbimage,sfind_pixel_fraction,options=None): if options is None: options = o f = fits.open(nonpbimage) imsizex = f[0].header['NAXIS1'] imsizey = f[0].header['NAXIS2'] f.close() kwargs={} if options['sfind_pixel_fraction']<1.0: lowerx,upperx = int(((1.0-sfind_pixel_fraction)/2.0)*imsizex),int(((1.0-sfind_pixel_fraction)/2.0)*imsizex + sfind_pixel_fraction*imsizex) lowery,uppery = int(((1.0-sfind_pixel_fraction)/2.0)*imsizey),int(((1.0-sfind_pixel_fraction)/2.0)*imsizey + sfind_pixel_fraction*imsizey) kwargs['trim_box']=(lowerx,upperx,lowery,uppery) if options['restart'] and os.path.isfile(catprefix +'.cat.fits'): warn('File ' + catprefix +'.cat.fits already exists, skipping source finding step') else: img = bdsm.process_image(pbimage, detection_image=nonpbimage, thresh_isl=4.0, thresh_pix=5.0, rms_box=(160,50), rms_map=True, mean_map='zero', ini_method='intensity', adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(60,15), group_by_isl=False, group_tol=10.0,output_opts=True, output_all=True, atrous_do=True,atrous_jmax=4, flagging_opts=True, flag_maxsize_fwhm=0.5,advanced_opts=True, ncores=options['NCPU'], blank_limit=None,**kwargs) img.write_catalog(outfile=catprefix +'.cat.fits',catalog_type='srl',format='fits',correct_proj='True') img.export_image(outfile=catprefix +'.rms.fits',img_type='rms',img_format='fits',clobber=True) img.export_image(outfile=catprefix +'.resid.fits',img_type='gaus_resid',img_format='fits',clobber=True) img.export_image(outfile=catprefix +'.pybdsmmask.fits',img_type='island_mask',img_format='fits',clobber=True) img.write_catalog(outfile=catprefix +'.cat.reg',catalog_type='srl',format='ds9',correct_proj='True')
def sfind_image(catprefix,pbimage,nonpbimage,sfind_pixel_fraction,options=None): if options is None: options = o f = fits.open(nonpbimage) imsizex = f[0].header['NAXIS1'] imsizey = f[0].header['NAXIS2'] f.close() kwargs={} if o['sfind_pixel_fraction']<1.0: lowerx,upperx = int(((1.0-sfind_pixel_fraction)/2.0)*imsizex),int(((1.0-sfind_pixel_fraction)/2.0)*imsizex + sfind_pixel_fraction*imsizex) lowery,uppery = int(((1.0-sfind_pixel_fraction)/2.0)*imsizey),int(((1.0-sfind_pixel_fraction)/2.0)*imsizey + sfind_pixel_fraction*imsizey) kwargs['trim_box']=(lowerx,upperx,lowery,uppery) if options['restart'] and os.path.isfile(catprefix +'.cat.fits'): warn('File ' + catprefix +'.cat.fits already exists, skipping source finding step') else: img = bdsm.process_image(pbimage, detection_image=nonpbimage, thresh_isl=4.0, thresh_pix=5.0, rms_box=(160,50), rms_map=True, mean_map='zero', ini_method='intensity', adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(60,15), group_by_isl=False, group_tol=10.0,output_opts=True, output_all=True, atrous_do=True,atrous_jmax=4, flagging_opts=True, flag_maxsize_fwhm=0.5,advanced_opts=True, blank_limit=None,**kwargs) img.write_catalog(outfile=catprefix +'.cat.fits',catalog_type='srl',format='fits',correct_proj='True') img.export_image(outfile=catprefix +'.rms.fits',img_type='rms',img_format='fits',clobber=True) img.export_image(outfile=catprefix +'.resid.fits',img_type='gaus_resid',img_format='fits',clobber=True) img.export_image(outfile=catprefix +'.pybdsmmask.fits',img_type='island_mask',img_format='fits',clobber=True) img.write_catalog(outfile=catprefix +'.cat.reg',catalog_type='srl',format='ds9',correct_proj='True')
def make_mask(image_name, mask_name=None, threshisl=5, atrous_do=False, rmsbox=(100,30), mask_combine=None): import sys, os import numpy as np from astropy.io import fits as pyfits import bdsf # wavelets are required to fit gaussians if atrous_do: stop_at = None else: stop_at = 'isl' # DO THE SOURCE DETECTION img = bdsf.process_image(image_name, rms_box=rmsbox, \ thresh_isl=int(threshisl), atrous_do=atrous_do, atrous_jmax=3, \ adaptive_rms_box=True, adaptive_thresh=100, rms_box_bright=(30,10), stop_at=stop_at, quiet=True, debug=False) # WRITE THE MASK FITS if mask_name == None: mask_name = image_name+'.newmask' if os.path.exists(mask_name): os.system('rm -r ' + mask_name) print('Making mask:', mask_name) img.export_image(img_type='island_mask', img_format='fits', outfile=mask_name) del img # do an pixel-by-pixel "OR" operation with a given mask if not mask_combine is None: print("Doing a pix-by-pix OR with %s." % mask_combine) with pyfits.open(mask_combine) as fits: data_comb = fits[0].data with pyfits.open(mask_name) as fits: data = fits[0].data assert data.shape() == data_comb.shape() data[(data_comb == 1.)] = 1. fits[0].data = data fits.writeto(mask_name, clobber=True) return mask_name
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Tue Jul 14 08:04:50 2020 @author: kabelo """ import bdsf import glob input_images = [] for image in glob.glob("*.fits"): input_images.append(image) for input_image in input_images: img = bdsf.process_image(input_image, rms_box=(20, 10)) img.write_catalog('./AbellOutput/', format='fits', catalog_type='srl') img.export_image('./AbellOutput/', img_type='gaus_resid') img.export_image('./AbellOutput/', img_type='gaus_model', outfile=input_image + '.model')
infile = 'mosaic-blanked.fits' if args.no_highres: catprefix = 'low-mosaic' infile = 'low-mosaic-blanked.fits' img = bdsm.process_image(infile, thresh_isl=4.0, thresh_pix=5.0, rms_box=(150, 15), rms_map=True, mean_map='zero', ini_method='intensity', adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(60, 15), group_by_isl=False, group_tol=10.0, output_opts=True, output_all=True, atrous_do=True, atrous_jmax=4, flagging_opts=True, flag_maxsize_fwhm=0.5, advanced_opts=True, blank_limit=None, frequency=restfrq) img.write_catalog(outfile=catprefix + '.cat.fits', catalog_type='srl', format='fits', correct_proj='True') img.export_image(outfile=catprefix + '.rms.fits',
def do_offsets(o): # o is the options file if o['mode']!='normal' and o['mode']!='test': raise NotImplementedError('Offsets called with mode '+o['mode']) image_root='image_full_ampphase_di_m.NS' method=o['method'] report('Determining astrometric offsets with method '+method+' in mode '+o['mode']) report('Merging downloaded catalogues') if os.path.isfile(method+'.fits'): warn('Merged file exists, reading from disk instead') data=Table.read(method+'.fits') else: if method=='pslocal': data=Table.read(method+'/'+method+'.txt',format='ascii') data['RA'].name='ra' data['DEC'].name='dec' data.write(method+'.fits') else: kwargs={} if 'panstarrs' in method: kwargs['rastr']='ramean' kwargs['decstr']='decmean' data=merge_cat(method,**kwargs) if o['mode']=='test': image_root+='_shift' method+='-test' report('Running PyBDSM on LOFAR image, please wait...') catfile=image_root+'.offset_cat.fits' gaulfile=catfile.replace('cat','gaul') if os.path.isfile(catfile): warn('Catalogue already exists, skipping pybdsf run') else: if o['mode']=='test': suffix='facetRestored' else: suffix='restored' pbimage=image_root+'.int.'+suffix+'.fits' nonpbimage=image_root+'.app.'+suffix+'.fits' img = bdsm.process_image(pbimage, detection_image=nonpbimage, thresh_isl=4.0, thresh_pix=5.0, rms_box=(150,15), rms_map=True, mean_map='zero', ini_method='intensity', adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(60,15), group_by_isl=False, group_tol=10.0,output_opts=True, output_all=True, atrous_do=False, flagging_opts=True, flag_maxsize_fwhm=0.5,advanced_opts=True, blank_limit=None) img.write_catalog(outfile=catfile,catalog_type='srl',format='fits',correct_proj='True') img.write_catalog(outfile=gaulfile,catalog_type='gaul',format='fits',correct_proj='True') lofar=Table.read(catfile) print len(lofar),'LOFAR sources before filtering' filter=(lofar['E_RA']*3600.0)<2.0 filter&=(lofar['E_DEC']*3600.0)<2.0 filter&=(lofar['Maj']*3600.0)<10 lofar=lofar[filter] print len(lofar),'LOFAR sources after filtering' regfile=image_root+'.tessel.reg' cra,cdec=get_centpos() report('Set up structure') NDir=np.load("image_dirin_SSD_m.npy.ClusterCat.npy").shape[0] oo=Offsets(method,n=NDir,imroot=image_root,cellsize=o['cellsize'],fitmethod=o['fit']) report('Label table') lofar_l=oo.r.add_facet_labels(lofar) report('Finding offsets') oo.find_offsets(lofar_l,data) report('Fitting offsets') oo.fit_offsets() report('Making plots and saving output') #oo.plot_fits(method+'-fits.pdf') oo.save_fits() oo.plot_offsets() if 'test' not in o['mode']: oo.save(method+'-fit_state.pickle') report('Making astrometry error map, please wait') oo.make_astrometry_map('astromap.fits',20) oo.offsets_to_facetshift('facet-offset.txt')
def main(image_name, mask_name, atrous_do=False, threshisl=0.0, threshpix=0.0, rmsbox=None, rmsbox_bright=(35, 7), iterate_threshold=False, adaptive_rmsbox=False, img_format='fits', threshold_format='float', trim_by=0.0, vertices_file=None, atrous_jmax=6, pad_to_size=None, skip_source_detection=False, region_file=None, nsig=1.0, reference_ra_deg=None, reference_dec_deg=None, cellsize_deg=0.000417, use_adaptive_threshold=False, adaptive_thresh=150.0): """ Make a clean mask and return clean threshold Parameters ---------- image_name : str Filename of input image from which mask will be made. If the image does not exist, a template image with center at (reference_ra_deg, reference_dec_deg) will be made internally mask_name : str Filename of output mask image atrous_do : bool, optional Use wavelet module of PyBDSF? threshisl : float, optional Value of thresh_isl PyBDSF parameter threshpix : float, optional Value of thresh_pix PyBDSF parameter rmsbox : tuple of floats, optional Value of rms_box PyBDSF parameter rmsbox_bright : tuple of floats, optional Value of rms_box_bright PyBDSF parameter iterate_threshold : bool, optional If True, threshold will be lower in 20% steps until at least one island is found adaptive_rmsbox : tuple of floats, optional Value of adaptive_rms_box PyBDSF parameter img_format : str, optional Format of output mask image (one of 'fits' or 'casa') threshold_format : str, optional Format of output threshold (one of 'float' or 'str_with_units') trim_by : float, optional Fraction by which the perimeter of the output mask will be trimmed (zeroed) vertices_file : str, optional Filename of file with vertices (must be a pickle file containing a dictionary with the vertices in the 'vertices' entry) atrous_jmax : int, optional Value of atrous_jmax PyBDSF parameter pad_to_size : int, optional Pad output mask image to a size of pad_to_size x pad_to_size skip_source_detection : bool, optional If True, source detection is not run on the input image region_file : str, optional Filename of region file in CASA format. If given, no mask image is made (the region file is used as the clean mask) nsig : float, optional Number of sigma of returned threshold value reference_ra_deg : float, optional RA for center of output mask image reference_dec_deg : float, optional Dec for center of output mask image cellsize_deg : float, optional Size of a pixel in degrees use_adaptive_threshold : bool, optional If True, use an adaptive threshold estimated from the negative values in the image adaptive_thresh : float, optional If adaptive_rmsbox is True, this value sets the threshold above which a source will use the small rms box Returns ------- result : dict Dict with nsig-sigma rms threshold """ if rmsbox is not None and type(rmsbox) is str: rmsbox = eval(rmsbox) if type(rmsbox_bright) is str: rmsbox_bright = eval(rmsbox_bright) if pad_to_size is not None and type(pad_to_size) is str: pad_to_size = int(pad_to_size) if type(atrous_do) is str: if atrous_do.lower() == 'true': atrous_do = True threshisl = 4.0 # override user setting to ensure proper source fitting else: atrous_do = False if type(iterate_threshold) is str: if iterate_threshold.lower() == 'true': iterate_threshold = True else: iterate_threshold = False if type(adaptive_rmsbox) is str: if adaptive_rmsbox.lower() == 'true': adaptive_rmsbox = True else: adaptive_rmsbox = False if type(skip_source_detection) is str: if skip_source_detection.lower() == 'true': skip_source_detection = True else: skip_source_detection = False if type(use_adaptive_threshold) is str: if use_adaptive_threshold.lower() == 'true': use_adaptive_threshold = True else: use_adaptive_threshold = False if reference_ra_deg is not None and reference_dec_deg is not None: reference_ra_deg = float(reference_ra_deg) reference_dec_deg = float(reference_dec_deg) if not os.path.exists(image_name): print('Input image not found. Making empty image...') if not skip_source_detection: print('ERROR: Source detection cannot be done on an empty image') sys.exit(1) if reference_ra_deg is not None and reference_dec_deg is not None: image_name = mask_name + '.tmp' make_template_image(image_name, reference_ra_deg, reference_dec_deg, cellsize_deg=float(cellsize_deg)) else: print( 'ERROR: if image not found, a refernce position must be given') sys.exit(1) trim_by = float(trim_by) atrous_jmax = int(atrous_jmax) threshpix = float(threshpix) threshisl = float(threshisl) nsig = float(nsig) adaptive_thresh = float(adaptive_thresh) threshold = 0.0 if not skip_source_detection: if vertices_file is not None: # Modify the input image to blank the regions outside of the polygon temp_img = pim.image(image_name) image_name += '.blanked' temp_img.saveas(image_name, overwrite=True) input_img = pim.image(image_name) data = input_img.getdata() vertices = read_vertices(vertices_file) RAverts = vertices[0] Decverts = vertices[1] xvert = [] yvert = [] for RAvert, Decvert in zip(RAverts, Decverts): pixels = input_img.topixel( [1, 1, Decvert * np.pi / 180.0, RAvert * np.pi / 180.0]) xvert.append(pixels[2]) # x -> Dec yvert.append(pixels[3]) # y -> RA poly = Polygon(xvert, yvert) # Find masked regions masked_ind = np.where(data[0, 0]) # Find distance to nearest poly edge and set to NaN those that # are outside the facet (dist < 0) dist = poly.is_inside(masked_ind[0], masked_ind[1]) outside_ind = np.where(dist < 0.0) if len(outside_ind[0]) > 0: data[0, 0, masked_ind[0][outside_ind], masked_ind[1][outside_ind]] = np.nan # Save changes input_img.putdata(data) if use_adaptive_threshold: # Get an estimate of the rms img = bdsf.process_image(image_name, mean_map='zero', rms_box=rmsbox, thresh_pix=threshpix, thresh_isl=threshisl, atrous_do=atrous_do, ini_method='curvature', thresh='hard', adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright, rms_map=True, quiet=True, atrous_jmax=atrous_jmax, stop_at='isl') # Find min and max pixels max_neg_val = abs(np.min(img.ch0_arr)) max_neg_pos = np.where(img.ch0_arr == np.min(img.ch0_arr)) max_pos_val = abs(np.max(img.ch0_arr)) max_pos_pos = np.where(img.ch0_arr == np.max(img.ch0_arr)) # Estimate new thresh_isl from min pixel value's sigma, but don't let # it get higher than 1/2 of the peak's sigma threshisl_neg = 2.0 * max_neg_val / img.rms_arr[max_neg_pos][0] max_sigma = max_pos_val / img.rms_arr[max_pos_pos][0] if threshisl_neg > max_sigma / 2.0: threshisl_neg = max_sigma / 2.0 # Use the new threshold only if it is larger than the user-specified one if threshisl_neg > threshisl: threshisl = threshisl_neg if iterate_threshold: # Start with given threshold and lower it until we get at least one island nisl = 0 while nisl == 0: img = bdsf.process_image(image_name, mean_map='zero', rms_box=rmsbox, thresh_pix=threshpix, thresh_isl=threshisl, atrous_do=atrous_do, ini_method='curvature', thresh='hard', adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright, rms_map=True, quiet=True, atrous_jmax=atrous_jmax) nisl = img.nisl threshpix /= 1.2 threshisl /= 1.2 if threshpix < 5.0: break else: img = bdsf.process_image(image_name, mean_map='zero', rms_box=rmsbox, thresh_pix=threshpix, thresh_isl=threshisl, atrous_do=atrous_do, ini_method='curvature', thresh='hard', adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright, rms_map=True, quiet=True, atrous_jmax=atrous_jmax) if img.nisl == 0: if region_file is None or region_file == '[]': print('No islands found. Clean mask cannot be made.') sys.exit(1) else: # Continue on and use user-supplied region file skip_source_detection = True threshold = nsig * img.clipped_rms # Check if there are large islands preset (indicating that multi-scale # clean is needed) has_large_isl = False for isl in img.islands: if isl.size_active > 100: # Assuming normal sampling, a size of 100 pixels would imply # a source of ~ 10 beams has_large_isl = True if (region_file is not None and region_file != '[]' and skip_source_detection): # Copy region file and return if source detection was not done os.system('cp {0} {1}'.format(region_file.strip('[]"'), mask_name)) if threshold_format == 'float': return {'threshold_5sig': threshold} elif threshold_format == 'str_with_units': # This is done to get around the need for quotes around strings in casapy scripts # 'casastr/' is removed by the generic pipeline return {'threshold_5sig': 'casastr/{0}Jy'.format(threshold)} elif not skip_source_detection: img.export_image(img_type='island_mask', mask_dilation=0, outfile=mask_name, img_format=img_format, clobber=True) if (vertices_file is not None or trim_by > 0 or pad_to_size is not None or (region_file is not None and region_file != '[]') or skip_source_detection): # Alter the mask in various ways if skip_source_detection: # Read the image mask_im = pim.image(image_name) else: # Read the PyBDSF mask mask_im = pim.image(mask_name) data = mask_im.getdata() coordsys = mask_im.coordinates() if reference_ra_deg is not None and reference_dec_deg is not None: values = coordsys.get_referencevalue() values[2][0] = reference_dec_deg / 180.0 * np.pi values[2][1] = reference_ra_deg / 180.0 * np.pi coordsys.set_referencevalue(values) imshape = mask_im.shape() del (mask_im) if pad_to_size is not None: imsize = pad_to_size coordsys['direction'].set_referencepixel([imsize / 2, imsize / 2]) pixmin = (imsize - imshape[2]) / 2 if pixmin < 0: print("The padded size must be larger than the original size.") sys.exit(1) pixmax = pixmin + imshape[2] data_pad = np.zeros((1, 1, imsize, imsize), dtype=np.float32) data_pad[0, 0, pixmin:pixmax, pixmin:pixmax] = data[0, 0] new_mask = pim.image('', shape=(1, 1, imsize, imsize), coordsys=coordsys) new_mask.putdata(data_pad) else: new_mask = pim.image('', shape=imshape, coordsys=coordsys) new_mask.putdata(data) data = new_mask.getdata() if skip_source_detection: # Mask all pixels data[:] = 1 if vertices_file is not None: # Modify the clean mask to exclude regions outside of the polygon vertices = read_vertices(vertices_file) RAverts = vertices[0] Decverts = vertices[1] xvert = [] yvert = [] for RAvert, Decvert in zip(RAverts, Decverts): try: pixels = new_mask.topixel([ 0, 1, Decvert * np.pi / 180.0, RAvert * np.pi / 180.0 ]) except: pixels = new_mask.topixel([ 1, 1, Decvert * np.pi / 180.0, RAvert * np.pi / 180.0 ]) xvert.append(pixels[2]) # x -> Dec yvert.append(pixels[3]) # y -> RA poly = Polygon(xvert, yvert) # Find masked regions masked_ind = np.where(data[0, 0]) # Find distance to nearest poly edge and unmask those that # are outside the facet (dist < 0) dist = poly.is_inside(masked_ind[0], masked_ind[1]) outside_ind = np.where(dist < 0.0) if len(outside_ind[0]) > 0: data[0, 0, masked_ind[0][outside_ind], masked_ind[1][outside_ind]] = 0 if trim_by > 0.0: sh = np.shape(data) margin = int(sh[2] * trim_by / 2.0) data[0, 0, 0:sh[2], 0:margin] = 0 data[0, 0, 0:margin, 0:sh[3]] = 0 data[0, 0, 0:sh[2], sh[3] - margin:sh[3]] = 0 data[0, 0, sh[2] - margin:sh[2], 0:sh[3]] = 0 if region_file is not None and region_file != '[]': # Merge the CASA regions with the mask casa_polys = read_casa_polys(region_file.strip('[]"'), new_mask) for poly in casa_polys: # Find unmasked regions unmasked_ind = np.where(data[0, 0] == 0) # Find distance to nearest poly edge and mask those that # are inside the casa region (dist > 0) dist = poly.is_inside(unmasked_ind[0], unmasked_ind[1]) inside_ind = np.where(dist > 0.0) if len(inside_ind[0]) > 0: data[0, 0, unmasked_ind[0][inside_ind], unmasked_ind[1][inside_ind]] = 1 # Save changes new_mask.putdata(data) if img_format == 'fits': new_mask.tofits(mask_name, overwrite=True) elif img_format == 'casa': new_mask.saveas(mask_name, overwrite=True) else: print( 'Output image format "{}" not understood.'.format(img_format)) sys.exit(1) if not skip_source_detection: if threshold_format == 'float': return { 'threshold_5sig': nsig * img.clipped_rms, 'multiscale': has_large_isl } elif threshold_format == 'str_with_units': # This is done to get around the need for quotes around strings in casapy scripts # 'casastr/' is removed by the generic pipeline return { 'threshold_5sig': 'casastr/{0}Jy'.format(nsig * img.clipped_rms), 'multiscale': has_large_isl } else: return {'threshold_5sig': '0.0'}
#!/usr/bin/python import bdsf import sys f = sys.argv[1] img = bdsf.process_image(f, advanced_opts=True, detection_image=f.replace('-pb',''), interactive=True, thresh_pix=5., thresh_isl=3., \ adaptive_rms_box=True, rms_box_bright=(100,30), adaptive_thresh=10.) img.write_catalog(format='fits',catalog_type='srl', clobber=True) img.write_catalog(format='ds9',catalog_type='srl', clobber=True) img.write_catalog(format='ds9',catalog_type='gaul', clobber=True) img.export_image(outfile=f+'_gaus_resid.fits', img_type='gaus_resid',clobber=True) img.export_image(outfile=f+'_gaus_model.fits', img_type='gaus_model',clobber=True) img.export_image(outfile=f+'_rms.fits', img_type='rms',clobber=True) img.export_image(outfile=f+'_mean.fits', img_type='mean',clobber=True)
def main(input_image, input_skymodel_pb, input_bright_skymodel_pb, output_root, vertices_file, threshisl=5.0, threshpix=7.5, rmsbox=(150, 50), rmsbox_bright=(35, 7), adaptive_rmsbox=True, use_adaptive_threshold=False, adaptive_thresh=75.0, beamMS=None, peel_bright=False): """ Filter the input sky model so that they lie in islands in the image Parameters ---------- input_image : str Filename of input image to use to detect sources for filtering. Ideally, this should be a flat-noise image (i.e., without primary-beam correction) input_skymodel_pb : str Filename of input makesourcedb sky model, with primary-beam correction input_bright_skymodel_pb : str Filename of input makesourcedb sky model of bright sources only, with primary- beam correction output_root : str Root of filename of output makesourcedb sky models. Output filenames will be output_root+'.apparent_sky.txt' and output_root+'.true_sky.txt' vertices_file : str Filename of file with vertices threshisl : float, optional Value of thresh_isl PyBDSF parameter threshpix : float, optional Value of thresh_pix PyBDSF parameter rmsbox : tuple of floats, optional Value of rms_box PyBDSF parameter rmsbox_bright : tuple of floats, optional Value of rms_box_bright PyBDSF parameter adaptive_rmsbox : tuple of floats, optional Value of adaptive_rms_box PyBDSF parameter use_adaptive_threshold : bool, optional If True, use an adaptive threshold estimated from the negative values in the image adaptive_thresh : float, optional If adaptive_rmsbox is True, this value sets the threshold above which a source will use the small rms box peel_bright : bool, optional If True, bright sources were peeled, so add then back before filtering """ if rmsbox is not None and isinstance(rmsbox, str): rmsbox = eval(rmsbox) if isinstance(rmsbox_bright, str): rmsbox_bright = eval(rmsbox_bright) adaptive_rmsbox = misc.string2bool(adaptive_rmsbox) use_adaptive_threshold = misc.string2bool(use_adaptive_threshold) if isinstance(beamMS, str): beamMS = misc.string2list(beamMS) peel_bright = misc.string2bool(peel_bright) # Try to set the TMPDIR evn var to a short path, to ensure we do not hit the length # limits for socket paths (used by the mulitprocessing module). We try a number of # standard paths (the same ones used in the tempfile Python library) old_tmpdir = os.environ["TMPDIR"] for tmpdir in ['/tmp', '/var/tmp', '/usr/tmp']: if os.path.exists(tmpdir): os.environ["TMPDIR"] = tmpdir break # Run PyBDSF to make a mask for grouping if use_adaptive_threshold: # Get an estimate of the rms by running PyBDSF to make an rms map img = bdsf.process_image(input_image, mean_map='zero', rms_box=rmsbox, thresh_pix=threshpix, thresh_isl=threshisl, thresh='hard', adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright, rms_map=True, quiet=True, stop_at='isl') # Find min and max pixels max_neg_val = abs(np.min(img.ch0_arr)) max_neg_pos = np.where(img.ch0_arr == np.min(img.ch0_arr)) max_pos_val = abs(np.max(img.ch0_arr)) max_pos_pos = np.where(img.ch0_arr == np.max(img.ch0_arr)) # Estimate new thresh_isl from min pixel value's sigma, but don't let # it get higher than 1/2 of the peak's sigma threshisl_neg = 2.0 * max_neg_val / img.rms_arr[max_neg_pos][0] max_sigma = max_pos_val / img.rms_arr[max_pos_pos][0] if threshisl_neg > max_sigma / 2.0: threshisl_neg = max_sigma / 2.0 # Use the new threshold only if it is larger than the user-specified one if threshisl_neg > threshisl: threshisl = threshisl_neg img = bdsf.process_image(input_image, mean_map='zero', rms_box=rmsbox, thresh_pix=threshpix, thresh_isl=threshisl, thresh='hard', adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright, atrous_do=True, atrous_jmax=3, rms_map=True, quiet=True) emptysky = False if img.nisl > 0: maskfile = input_image + '.mask' img.export_image(outfile=maskfile, clobber=True, img_type='island_mask') # Construct polygon needed to trim the mask to the sector header = pyfits.getheader(maskfile, 0) w = wcs.WCS(header) RAind = w.axis_type_names.index('RA') Decind = w.axis_type_names.index('DEC') vertices = misc.read_vertices(vertices_file) RAverts = vertices[0] Decverts = vertices[1] verts = [] for RAvert, Decvert in zip(RAverts, Decverts): ra_dec = np.array([[0.0, 0.0, 0.0, 0.0]]) ra_dec[0][RAind] = RAvert ra_dec[0][Decind] = Decvert verts.append((w.wcs_world2pix(ra_dec, 0)[0][RAind], w.wcs_world2pix(ra_dec, 0)[0][Decind])) hdu = pyfits.open(maskfile, memmap=False) data = hdu[0].data # Rasterize the poly data_rasertize = data[0, 0, :, :] data_rasertize = misc.rasterize(verts, data_rasertize) data[0, 0, :, :] = data_rasertize hdu[0].data = data hdu.writeto(maskfile, overwrite=True) # Now filter the sky model using the mask made above if len(beamMS) > 1: # Select the best MS for the beam attenuation ms_times = [] for ms in beamMS: tab = pt.table(ms, ack=False) ms_times.append(np.mean(tab.getcol('TIME'))) tab.close() ms_times_sorted = sorted(ms_times) mid_time = ms_times_sorted[int(len(ms_times)/2)] beam_ind = ms_times.index(mid_time) else: beam_ind = 0 try: s = lsmtool.load(input_skymodel_pb, beamMS=beamMS[beam_ind]) except astropy.io.ascii.InconsistentTableError: emptysky = True if peel_bright: try: # If bright sources were peeled before imaging, add them back s_bright = lsmtool.load(input_bright_skymodel_pb, beamMS=beamMS[beam_ind]) # Rename the bright sources, removing the '_sector_*' added previously # (otherwise the '_sector_*' text will be added every iteration, # eventually making for very long source names) new_names = [name.split('_sector')[0] for name in s_bright.getColValues('Name')] s_bright.setColValues('Name', new_names) if not emptysky: s.concatenate(s_bright) else: s = s_bright emptysky = False except astropy.io.ascii.InconsistentTableError: pass if not emptysky: s.select('{} == True'.format(maskfile)) # keep only those in PyBDSF masked regions if len(s) == 0: emptysky = True else: # Write out apparent and true-sky models del(img) # helps reduce memory usage s.group(maskfile) # group the sky model by mask islands s.write(output_root+'.true_sky.txt', clobber=True) s.write(output_root+'.apparent_sky.txt', clobber=True, applyBeam=True) else: emptysky = True if emptysky: # No sources cleaned/found in image, so just make a dummy sky model with single, # very faint source at center dummylines = ["Format = Name, Type, Patch, Ra, Dec, I, SpectralIndex, LogarithmicSI, " "ReferenceFrequency='100000000.0', MajorAxis, MinorAxis, Orientation\n"] ra, dec = img.pix2sky((img.shape[-2]/2.0, img.shape[-1]/2.0)) if ra < 0.0: ra += 360.0 ra = misc.ra2hhmmss(ra) sra = str(ra[0]).zfill(2)+':'+str(ra[1]).zfill(2)+':'+str("%.6f" % (ra[2])).zfill(6) dec = misc.dec2ddmmss(dec) decsign = ('-' if dec[3] < 0 else '+') sdec = decsign+str(dec[0]).zfill(2)+'.'+str(dec[1]).zfill(2)+'.'+str("%.6f" % (dec[2])).zfill(6) dummylines.append(',,p1,{0},{1}\n'.format(sra, sdec)) dummylines.append('s0c0,POINT,p1,{0},{1},0.00000001,' '[0.0,0.0],false,100000000.0,,,\n'.format(sra, sdec)) with open(output_root+'.apparent_sky.txt', 'w') as f: f.writelines(dummylines) with open(output_root+'.true_sky.txt', 'w') as f: f.writelines(dummylines) # Set the TMPDIR env var back to its original value os.environ["TMPDIR"] = old_tmpdir
def run_bdsf(image, output_dir, argfile, output_format): ''' Run PyBDSF on an image Keyword arguments: image -- Name of image argfile -- Input json file containing arguments for bdsf functions ''' imname = os.path.join(output_dir, os.path.basename(image).split('.')[0]) path = Path(__file__).parent / argfile with open(path) as f: args_dict = json.load(f) # Fix json stupidness args_dict['process_image']['rms_box'] = ast.literal_eval( args_dict['process_image']['rms_box']) args_dict['process_image']['rms_box_bright'] = ast.literal_eval( args_dict['process_image']['rms_box_bright']) img = bdsf.process_image(image, **args_dict['process_image']) for img_type in args_dict['export_image']: if args_dict['export_image'][img_type]: img.export_image(outfile=imname + '_' + img_type + '.fits', clobber=True, img_type=img_type) outcat = None for of in output_format: fmt = of.lower().split(':') if len(fmt) == 1: fmt = fmt[0] cat_type = 'srl' if len(fmt) == 2: fmt, cat_type = fmt if fmt == 'ds9': outcatalog = imname + '_' + cat_type + '_bdsfcat.ds9.reg' img.write_catalog(outfile=outcatalog, format=fmt, catalog_type=cat_type, clobber=True) elif fmt == 'kvis': outcatalog = imname + '_bdsfcat.kvis.ann' img.write_catalog(outfile=outcatalog, format=fmt, catalog_type='gaul', clobber=True) elif fmt == 'star': outcatalog = imname + '_bdsfcat.star' img.write_catalog(outfile=outcatalog, format=fmt, catalog_type='gaul', clobber=True) else: outcatalog = imname + '_' + cat_type + '_bdsfcat.' + fmt img.write_catalog(outfile=outcatalog, format=fmt, catalog_type=cat_type, clobber=True) if fmt == 'fits' and cat_type == 'srl': outcat = outcatalog return outcat, img
# configure trimbox if args.trimbox is not None: with fits.open(fname) as hdu: naxis1 = hdu[0].header["NAXIS1"] naxis2 = hdu[0].header["NAXIS1"] trim_box = (int(naxis1 / 2) - args.trimbox, int(naxis1 / 2) + args.trimbox, int(naxis2 / 2) - args.trimbox, int(naxis2 / 2) + args.trimbox) else: trim_box = None # run bdsf try: img = bdsf.process_image(fname, quiet=args.silence, trim_box=trim_box, blank_limit=args.blank_limit) except RuntimeError: continue # save catalog to ASCII file outfiles.append(out_fname) echo("::: saving {} :::".format(out_fname)) bdsf.output.write_ascii_list(img, filename=out_fname, clobber=args.overwrite) # stop if no source provided if args.source is None: echo( "can't proceed with source extraction because args.source is not provided"
def main (vis,strategy='P30,P30,P30,A500,A450,A400',startmod='',ith=5.0,\ bandwidth='8MHz',goodness=2.,minuvw=50.0,robust=-1.0): ## format arguments strategy = str(strategy) startmod = str(startmod) ith = float(ith) bandwidth = str(bandwidth) minuvw = float(minuvw) * 1000.0 # convert km -> m robust = float(robust) ## process arguments vis = vis.rstrip('/') vis = vis.split('/')[-1] strategy = strategy.split(',') bw_val = '' bw_unit = '' for c in bandwidth: try: float(c) bw_val = bw_val + c except ValueError: bw_unit = bw_unit + c if bw_unit == 'MHz': bw_val = float(bw_val) * 1e6 ## get bandwidth of vis spec_info = casatb.table(vis + '::SPECTRAL_WINDOW') total_bw = spec_info.getcol('TOTAL_BANDWIDTH')[0] num_chan = spec_info.getcol('NUM_CHAN')[0] spec_info.close() if total_bw < bw_val: wsclean_chans = 0 mfs = '' nchan = 0 else: wsclean_chans = int(np.ceil(total_bw / bw_val)) mfs = '-MFS' nchan = num_chan / wsclean_chans ## make a working directory and go there tmp_dir = 'loop3_' + vis.rstrip('.ms').rstrip('.MS') os.system('mkdir %s' % tmp_dir) os.chdir(tmp_dir) os.system('mv ../%s .' % vis) import bdsf prevstat = 0.0 cohlength = 2.0E6 strategy_type = [] for i in strategy: strategy_type.append(i[0]) ploop, nloop, snver = strategy_type.count('P'), len(strategy), 0 # # PHASE CALIBRATION - run through ploop iterations, exiting if we have convergence # for iloop in range(ploop): fitsmask = vis + '_%02d-mask.fits' % (iloop - 1) if iloop else '' if startmod == '' or iloop: pstr = '******* PHASE LOOP %d running wsclean ************' % iloop loop3log(vis, pstr + '\n') imagr(vis,minuvwm=minuvw,robust=robust,cellsize='0.05asec',domultiscale=True,\ outname=vis+'_%02d'%iloop,channelsout=wsclean_chans,\ fitsmask=fitsmask,dolocalrms=True,maxuvwm=cohlength) else: # Need something here to produce an image from startmod pass # check if there's a source thisstat = measure_statistic2(vis + '_%02d%s-image.fits' % (iloop, mfs)) if thisstat < goodness: pstr = 'SNR is %f, breaking out of loop.' % thisstat loop3log(vis, pstr + '\n') montage_plot('*MFS-image.fits', imscale=0.65, nup='4x2', plot_resid=False) return (0) pstr = '******* PHASE LOOP %d making mask %s_%02d%s-image.fits ********' % ( iloop, vis, iloop, mfs) loop3log(vis, pstr + '\n') stdout = sys.stdout sys.stdout = open('bdsf_chunterings', 'a') img = bdsf.process_image('%s_%02d%s-image.fits' % (vis, iloop, mfs), atrous_do=True, thresh_isl=ith) sys.stdout.close() sys.stdout = stdout img.export_image(img_type='island_mask', outfile='%s_%02d-mask.fits' % (vis, iloop)) # exit loop if clean finishing pstr = '******* PHASE LOOP %d goodness stat %f ************' % ( iloop, thisstat) loop3log(vis, pstr + '\n') if thisstat - prevstat < 0.01: pstr = '****** EXITING PHASE CAL with diff %f *********' % ( thisstat - prevstat) loop3log(vis, pstr + '\n') break else: prevstat = thisstat imagr(vis,minuvwm=minuvw,robust=robust,dopredict=True,fitsmask=fitsmask,\ autothreshold=3,dolocalrms=True,\ outname=vis+'_%02d%s'%(iloop,mfs)) pstr = '******* PHASE LOOP %d making new cal file %s ************' % ( iloop, vis + '_%02d' % iloop) loop3log(vis, pstr + '\n') caltype, sol0 = strategy[iloop][0], float(strategy[iloop][1:]) coh, cohlength = selfcal(vis,minuvw,robust,model='MODEL',incol='DATA',\ outcol='CORRECTED_DATA',outcal_root=vis+'_%02d'%iloop,\ caltype=caltype,init_sol=sol0,nchan=nchan) snver = iloop pstr='******** END PHASE LOOP %d - coherence on %.1f km **********' % \ (iloop,cohlength/1000.) loop3log(vis, pstr + '\n') # Exit at this point if we are not doing amplitude cal if ploop == nloop: exit() # # If we are doing amplitude calibration, we now need to apply the # calibration and write a new MS with a DATA column visA = vis + '_A' # delete all existing files beginning with vis+'_A' os.system('rm -fr %s*' % visA) pstr = '****** APPLYING CALIBRATION TABLE %d\n' % snver loop3log(vis, pstr + '\n') applycal_split(vis, visA, 'sol000', '%s_%02d_c0.h5' % (vis, snver)) init_fitsmask = vis + '_%02d-mask.fits' % iloop init_img = vis + '_%02d%s-image.fits' % (iloop, mfs) pred_img = vis + '_%02d%s' % (iloop, mfs) for iloop in range(ploop, nloop): fitsmask = init_fitsmask if iloop == ploop else visA + '_%02d-mask.fits' % ( iloop - 1) pstr = '******* AMPLITUDE LOOP %d running wsclean ************' % iloop loop3log(vis, pstr + '\n') imagr(visA,minuvwm=minuvw,robust=robust,cellsize='0.05asec',domultiscale=True,\ outname=visA+'_%02d'%iloop,channelsout=wsclean_chans,\ fitsmask=fitsmask,dolocalrms=True,maxuvwm=cohlength) ## check if there's a source thisstat = measure_statistic2(visA + '_%02d%s-image.fits' % (iloop, mfs)) if thisstat < goodness: pstr = 'SNR is %f, breaking out of loop.' % thisstat loop3log(vis, pstr + '\n') montage_plot('*MFS-image.fits', imscale=0.65, nup='4x2', plot_resid=True) return (0) image_bdsf = '%s_%02d%s-image.fits' % (visA, iloop, mfs) pstr = '******* AMPLITUDE LOOP %d making mask %s_%02d%s-image.fits ************' % ( iloop, visA, iloop, mfs) loop3log(vis, pstr + '\n') img = bdsf.process_image(image_bdsf, atrous_do=True, thresh_isl=ith) img.export_image(img_type='island_mask', outfile='%s_%02d-mask.fits' % (visA, iloop)) pstr = '******* AMPLITUDE LOOP %d goodness stat %f ************' % ( iloop, thisstat) loop3log(vis, pstr + '\n') if iloop != ploop and thisstat - prevstat < 0.01: pstr = '****** EXITING AMPLITUDE CAL with diff %f *********' % ( thisstat - prevstat) loop3log(vis, pstr + '\n') break else: prevstat = thisstat imagr(visA,minuvwm=minuvw,dopredict=True,fitsmask=fitsmask,\ autothreshold=3,dolocalrms=True,robust=robust,\ outname=visA+'_%02d%s'%(iloop,mfs)) pstr = '******* AMPLITUDE LOOP %d making new cal file %s ************' % ( iloop, visA + '_%02d' % iloop) loop3log(vis, pstr + '\n') caltype, sol0 = strategy[iloop][0], float(strategy[iloop][1:]) coh,cohlength = selfcal(visA,minuvw,robust,model='MODEL',incol='DATA',\ outcol='CORRECTED_DATA',outcal_root=visA+'_%02d'%iloop,\ caltype=caltype,init_sol=sol0,nchan=nchan) pstr='******** END AMPLITUDE LOOP %d - coherence on %.1f km **********' % \ (iloop,cohlength/1000.) loop3log(vis, pstr + '\n') fitsmask = init_fitsmask if iloop == ploop else visA + '_%02d-mask.fits' % ( iloop - 1) imagr(visA,minuvwm=minuvw,cellsize='0.05asec',domultiscale=True,\ outname=visA+'_final',channelsout=wsclean_chans,robust=robust,\ fitsmask=fitsmask,dolocalrms=True) ## make a model from the final image final_im = glob.glob('*final*image.fits') if len(final_im) > 1: tmp = [a for a in final_im if 'MFS' in a] final_im = tmp img = bdsf.process_image(final_im[0], atrous_do=True, thresh_isl=ith) skyfile = final_im[0].replace('fits', 'skymodel') img.write_catalog(outfile=skyfile, bbs_patches='single', catalog_type='gaul', format='bbs') ## convert it to a sourcedb ss = "makesourcedb in=%s out=%s format='<'" % ( skyfile, skyfile.replace('skymodel', 'sky')) os.system(ss) ## plot things like solutions make_plots(vis) make_plots(visA) ## If we got to this point, self-cal has successfully completed # montage_plot( '*MFS-image.fits', imscale=0.65, nup='4x2', plot_resid=True) pngfile, h5files = cleanup(vis) for h5file in h5files: os.system('mv %s ../' % h5file) # os.system('mv *.pdf ../') # os.system('mv *.png ../') os.system('mv *skymodel ../') os.system('mv *sky ../') os.system('mv %s ../' % vis) print 'Output calibration tables', h5files #return pngfile,h5files return 0
def aplpy_plots(infits, docut=2.0, outpng='', nolabel=False, crms=3.0, noshift=False, margin=1.7): ## open the file to get the data and the header information print 'Plotting ', infits hdul = pyfits.open(infits) a = hdul[0].data.squeeze() h = hdul[0].header nx, ny = h['NAXIS1'], h['NAXIS2'] field_radius = h['CDELT2'] * ny / 2.0 trms, tmax = np.array([]), np.array([]) for i in range(10): x1, x2 = int(0.1 * i * nx), int(0.1 * (i + 1) * nx - 1) for j in range(10): y1, y2 = int(0.1 * j * ny), int(0.1 * (j + 1) * ny - 1) trms = np.append(trms, np.std(a[y1:y2, x1:x2])) tmax = np.append(tmax, np.std(a[y1:y2, x1:x2])) rms = np.nanmedian(trms) vmin, vmax = np.nanmin(a), np.nanmax(a) ## calculate the required sigma reqsig = stats.norm.ppf(1 - 0.5 / float(nx * ny)) + (2.0 if docut < -1.0 else docut) ## run bdsf to get a gaussian list img = bdsf.process_image(infits, thresh_isl=reqsig) img.write_catalog(outfile='temp.gaul', catalog_type='gaul', format='ascii', clobber=True) ## read in the file s = ascii.read('temp.gaul', format='commented_header', header_start=4) print s ## make a plot gc = aplpy.FITSFigure(infits) pixra, pixdec = np.mean(s['Xposn']), np.mean(s['Yposn']) dec = h['CRVAL2'] + h['CDELT2'] * (pixdec - h['CRPIX2']) cosdec = np.cos(np.deg2rad(dec)) ra = h['CRVAL1'] + h['CDELT1'] * (pixra - h['CRPIX1']) / cosdec deccen = h['CRVAL2'] + h['CDELT2'] * (0.5 * ny - h['CRPIX2']) cosdeccen = np.cos(np.deg2rad(deccen)) racen = h['CRVAL1'] + h['CDELT1'] * (0.5 * nx - h['CRPIX1']) / cosdeccen pix_range = max(s['Xposn'].max() - s['Xposn'].min(), s['Yposn'].max() - s['Yposn'].min()) try: deg_range = max(margin * h['CDELT2'] * pix_range, 0.1 * ny * h['CDELT2']) except: deg_range = 1. / 60. print 'Error on deg_range, pix_range %f' % pix_range if docut < -1.0: deg_range = -2.0 * docut / 3600. print 'Range is %.1f arcsec, %.1f pix\n' % (1800.0 * deg_range, pix_range) if noshift: gc.recenter(racen, deccen, 0.5 * deg_range) else: gc.recenter(ra, dec, 0.5 * deg_range) gc.set_tick_color('black') gc.show_colorscale(cmap=matplotlib.cm.gray_r, vmin=vmin, vmax=vmax) levels, tlevels = [vmax], vmax while tlevels > crms * rms: tlevels /= np.sqrt(2) levels.append(tlevels) gc.show_contour(levels=np.sort(levels)) if not nolabel: bstr = '' for i in range(len(h)): try: if 'BMAJ' in h[i] and 'AIPS' in h[i]: bmaj = 3600. * float(h[i].split('BMAJ=')[1].split()[0]) bmin = 3600. * float(h[i].split('BMIN=')[1].split()[0]) bstr = 'beam %.1fx%.1f' % (bmaj, bmin) except: pass if rms > 0.1: gc.add_label(0.5,0.05,'Peak %.1f, rms %.1f Jy %s'%\ (vmax,rms,bstr),relative=True,size=14) elif rms > 1.e-4: gc.add_label(0.5,0.05,'Peak %.1f, rms %.1f mJy %s'%\ (vmax*1000.,rms*1000.,bstr),relative=True,size=14) else: gc.add_label(0.5,0.05,'Peak %.1f, rms %.1f uJy %s'%\ (vmax*1.e6,rms*1.e6,bstr),relative=True,size=14) if outpng == '': outpng = infits.replace('fits', 'png') gc.save(outpng)
# Copyright (C) 2019 - Francesco de Gasperin # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import bdsf import sys f = sys.argv[1] img = bdsf.process_image(f, advanced_opts=True, detection_image=f.replace('-pb',''), interactive=True, thresh_pix=5., thresh_isl=3., \ adaptive_rms_box=True, rms_box_bright=(100,30), adaptive_thresh=10.) img.write_catalog(format='fits',catalog_type='srl', clobber=True) img.write_catalog(format='ds9',catalog_type='srl', clobber=True) img.write_catalog(format='ds9',catalog_type='gaul', clobber=True) img.export_image(outfile=f+'_gaus_resid.fits', img_type='gaus_resid',clobber=True) img.export_image(outfile=f+'_gaus_model.fits', img_type='gaus_model',clobber=True) img.export_image(outfile=f+'_rms.fits', img_type='rms',clobber=True) img.export_image(outfile=f+'_mean.fits', img_type='mean',clobber=True)
def run_bootstrap(o): colname='DATA_DI_CORRECTED' if o['mslist'] is None: die('MS list must be specified') if o['logging'] is not None and not os.path.isdir(o['logging']): os.mkdir(o['logging']) # check the data supplied if o['frequencies'] is None or o['catalogues'] is None: die('Frequencies and catalogues options must be specified') if "DDF_PIPELINE_CATALOGS" not in os.environ.keys(): warn("You need to define the environment variable DDF_PIPELINE_CATALOGS where your catalogs are located") sys.exit(2) o["tgss"]=o["tgss"].replace("$$",os.environ["DDF_PIPELINE_CATALOGS"]) o["catalogues"]=[l.replace("$$",os.environ["DDF_PIPELINE_CATALOGS"]) for l in o["catalogues"]] lCat=o["catalogues"]+[o["tgss"]] for fCat in lCat: if not os.path.isfile(fCat): warn("Catalog %s does not exist"%fCat) sys.exit(2) cl=len(o['catalogues']) if o['names'] is None: o['names']=[os.path.basename(x).replace('.fits','') for x in o['catalogues']] if o['radii'] is None: o['radii']=[10]*cl if o['groups'] is None: o['groups']=range(cl) if (len(o['frequencies'])!=cl or len(o['radii'])!=cl or len(o['names'])!=cl or len(o['groups'])!=cl): die('Names, groups, radii and frequencies entries must be the same length as the catalogue list') low_uvrange=[o['image_uvmin'],2.5*206.0/o['low_psf_arcsec']] if o['low_imsize'] is not None: low_imsize=o['low_imsize'] # allow over-ride else: low_imsize=o['imsize']*o['cellsize']/o['low_cell'] low_robust=o['low_robust'] # Clear the shared memory run('CleanSHM.py',dryrun=o['dryrun']) # We use the individual ms in mslist. m=MSList(o['mslist']) Uobsid = set(m.obsids) for obsid in Uobsid: warn('Running bootstrap for obsid %s' % obsid) freqs=[] omslist=[] for ms,ob,f in zip(m.mss,m.obsids,m.freqs): if ob==obsid: omslist.append(ms) freqs.append(f) if len(freqs)<4: die('Not enough frequencies to bootstrap. Check your mslist or MS naming scheme') # sort to work in frequency order freqs,omslist = (list(x) for x in zip(*sorted(zip(freqs, omslist), key=lambda pair: pair[0]))) for f,ms in zip(freqs,omslist): print ms,f # generate the sorted input mslist with open('temp_mslist.txt','w') as f: for line in omslist: f.write(line+'\n') # Clean in cube mode # As for the main pipeline, first make a dirty map ddf_image('image_bootstrap_'+obsid+'_init','temp_mslist.txt', cleanmask=None,cleanmode='SSD',ddsols='DDS0', applysols='P',majorcycles=0,robust=low_robust, uvrange=low_uvrange,beamsize=o['low_psf_arcsec'], imsize=low_imsize,cellsize=o['low_cell'], options=o,colname=colname,automask=True, automask_threshold=15,smooth=True,cubemode=True, conditional_clearcache=True) external_mask='bootstrap_external_mask.fits' make_external_mask(external_mask,'image_bootstrap_'+obsid+'_init.dirty.fits',use_tgss=True,clobber=False,cellsize='low_cell',options=o) # Deep SSD clean with this external mask and automasking ddf_image('image_bootstrap_'+obsid,'temp_mslist.txt', cleanmask=external_mask,reuse_psf=True,reuse_dirty=True, cleanmode='SSD',ddsols='DDS0',applysols='P', majorcycles=5,robust=low_robust,uvrange=low_uvrange, beamsize=o['low_psf_arcsec'],imsize=low_imsize, cellsize=o['low_cell'],options=o, colname=colname,automask=True, automask_threshold=15,smooth=True,cubemode=True, conditional_clearcache=False) if os.path.isfile('image_bootstrap_'+obsid+'.cube.int.restored.pybdsm.srl'): warn('Source list exists, skipping source extraction') else: warn('Running PyBDSM, please wait...') img=bdsm.process_image('image_bootstrap_'+obsid+'.cube.int.restored.fits',thresh_pix=5,rms_map=True,atrous_do=True,atrous_jmax=2,group_by_isl=True,rms_box=(80,20), adaptive_rms_box=True, adaptive_thresh=80, rms_box_bright=(35,7),mean_map='zero',spectralindex_do=True,specind_maxchan=1,debug=True,kappa_clip=3,flagchan_rms=False,flagchan_snr=False,incl_chan=True,spline_rank=1) # Write out in ASCII to work round bug in pybdsm img.write_catalog(catalog_type='srl',format='ascii',incl_chan='true') img.export_image(img_type='rms',img_format='fits') from make_fitting_product import make_catalogue import fitting_factors import find_outliers # generate the fitting product if os.path.isfile(obsid+'crossmatch-1.fits'): warn('Crossmatch table exists, skipping crossmatch') else: t = pt.table(omslist[0]+ '/FIELD', readonly=True, ack=False) direction = t[0]['PHASE_DIR'] ra, dec = direction[0] if (ra<0): ra+=2*np.pi ra*=180.0/np.pi dec*=180.0/np.pi cats=zip(o['catalogues'],o['names'],o['groups'],o['radii']) make_catalogue('image_bootstrap_'+obsid+'.cube.int.restored.pybdsm.srl',ra,dec,2.5,cats,outnameprefix=obsid) freqlist=open(obsid+'frequencies.txt','w') for n,f in zip(o['names'],o['frequencies']): freqlist.write('%f %s_Total_flux %s_E_Total_flux False\n' % (f,n,n)) for i,f in enumerate(freqs): freqlist.write('%f Total_flux_ch%i E_Total_flux_ch%i True\n' % (f,i+1,i+1)) freqlist.close() # Now call the fitting code if os.path.isfile(obsid+'crossmatch-results-1.npy'): warn('Results 1 exists, skipping first fit') else: fitting_factors.run_all(1, name=obsid) nreject=-1 # avoid error if we fail somewhere if os.path.isfile(obsid+'crossmatch-2.fits'): warn('Second crossmatch exists, skipping outlier rejection') else: nreject=find_outliers.run_all(1, name=obsid) if os.path.isfile(obsid+'crossmatch-results-2.npy'): warn('Results 2 exists, skipping second fit') else: if nreject==0: shutil.copyfile(obsid+'crossmatch-results-1.npy',obsid+'crossmatch-results-2.npy') if os.path.isfile(obsid+'crossmatch-results-2.npy'): warn('Results 2 exists, skipping first fit') else: fitting_factors.run_all(2, name=obsid) # Now apply corrections if o['full_mslist'] is None: die('Need big mslist to apply corrections') if not(o['dryrun']): warn('Applying corrections to MS list') scale=np.load(obsid+'crossmatch-results-2.npy')[:,0] # InterpolatedUS gives us linear interpolation between points # and extrapolation outside it spl = InterpolatedUnivariateSpline(freqs, scale, k=1) bigmslist=[s.strip() for s in open(o['full_mslist']).readlines()] obigmslist = [ms for ms in bigmslist if obsid in ms] for ms in obigmslist: t = pt.table(ms) try: dummy=t.getcoldesc('SCALED_DATA') except RuntimeError: dummy=None t.close() if dummy is not None: warn('Table '+ms+' has already been corrected, skipping') else: # in this version we need to scale both the original data and the data in colname t = pt.table(ms+'/SPECTRAL_WINDOW', readonly=True, ack=False) frq=t[0]['REF_FREQUENCY'] factor=spl(frq) print frq,factor t=pt.table(ms,readonly=False) desc=t.getcoldesc(o['colname']) desc['name']='SCALED_DATA' t.addcols(desc) d=t.getcol(o['colname']) d*=factor t.putcol('SCALED_DATA',d) try: dummy=t.getcoldesc(colname) except RuntimeError: dummy=None if dummy is not None: desc=t.getcoldesc(colname) newname=colname+'_SCALED' desc['name']=newname t.addcols(desc) d=t.getcol(colname) d*=factor t.putcol(newname,d) t.close() if os.path.isfile('image_bootstrap.app.mean.fits'): warn('Mean bootstrap image exists, not creating it') else: warn('Creating mean bootstrap image') hdus=[] for obsid in Uobsid: hdus.append(fits.open('image_bootstrap_'+obsid+'.app.restored.fits')) for i in range(1,len(Uobsid)): hdus[0][0].data+=hdus[i][0].data hdus[0][0].data/=len(Uobsid) hdus[0].writeto('image_bootstrap.app.mean.fits')
def hybridloops(vis, strategy=['P32'] * 3 + ['A1800', 'A1200', 'A600'], startmod='', ith=5.0): import bdsf prevstat = 0.0 cohlength = 2.0E6 strategy_type = [] for i in strategy: strategy_type.append(i[0]) ploop, nloop, snver = strategy_type.count('P'), len(strategy), 0 # # PHASE CALIBRATION - run through ploop iterations, exiting if we have convergence # for iloop in range(ploop): fitsmask = vis + '_%02d-mask.fits' % (iloop - 1) if iloop else '' if startmod == '' or iloop: pstr = '******* PHASE LOOP %d running wsclean ************' % iloop loop3_service.loop3log(vis, pstr + '\n') loop3_service.imagr(vis,cellsize='0.1asec',domultiscale=True,\ outname=vis+'_%02d'%iloop,dojoinchannels=True,channelsout=8,robust=0,\ fitsmask=fitsmask,dolocalrms=True,maxuvwm=cohlength) else: # Need something here to produce an image from startmod pass pstr = '******* PHASE LOOP %d making mask %s_%02d-MFA-image.fits ********' % ( iloop, vis, iloop) loop3_service.loop3log(vis, pstr + '\n') stdout = sys.stdout sys.stdout = open('bdsf_chunterings', 'a') img = bdsf.process_image('%s_%02d-MFS-image.fits' % (vis, iloop), atrous_do=True, thresh_isl=ith) sys.stdout.close() sys.stdout = stdout img.export_image(img_type='island_mask', outfile='%s_%02d-mask.fits' % (vis, iloop)) thisstat = measure_statistic(vis + '_%02d-MFS-image.fits' % iloop) ####### need a line here to bomb out if no coherence # exit loop if clean finishing pstr = '******* PHASE LOOP %d goodness stat %f ************' % ( iloop, thisstat) loop3_service.loop3log(vis, pstr + '\n') if thisstat - prevstat < 0.01: pstr = '****** EXITING PHASE CAL with diff %f *********' % ( thisstat - prevstat) loop3_service.loop3log(vis, pstr + '\n') break else: prevstat = thisstat loop3_service.imagr(vis,dopredict=True,fitsmask=fitsmask,autothreshold=2.5,dolocalrms=True,\ robust=0,outname=vis+'_%02d-MFS'%iloop) pstr = '******* PHASE LOOP %d making new cal file %s ************' % ( iloop, vis + '_%02d' % iloop) loop3_service.loop3log(vis, pstr + '\n') caltype, sol0 = strategy[iloop][0], float(strategy[iloop][1:]) coh, cohlength = selfcal(vis,model='MODEL',incol='DATA',outcol='CORRECTED_DATA',\ outcal_root=vis+'_%02d'%iloop,caltype=caltype,init_sol=sol0) snver = iloop pstr='******** END PHASE LOOP %d - coherence on %.1f km **********' % \ (iloop,cohlength/1000.) loop3_service.loop3log(vis, pstr + '\n') # Exit at this point if we are not doing amplitude cal if ploop == nloop: exit() # # If we are doing amplitude calibration, we now need to apply the # calibration and write a new MS with a DATA column visA = vis + '_A' # delete all existing files beginning with vis+'_A' os.system('rm -fr %s*' % visA) pstr = '****** APPLYING CALIBRATION TABLE %d\n' % snver loop3_service.loop3log(vis, pstr + '\n') applycal_split(vis, visA, 'sol000', '%s_%02d_c0.h5' % (vis, snver)) init_fitsmask = vis + '_%02d-mask.fits' % iloop init_img = vis + '_%02d-MFS-image.fits' % iloop pred_img = vis + '_%02d-MFS' % iloop for iloop in range(ploop, nloop): fitsmask = init_fitsmask if iloop == ploop else visA + '_%02d-mask.fits' % ( iloop - 1) pstr = '******* AMPLITUDE LOOP %d running wsclean ************' % iloop loop3_service.loop3log(vis, pstr + '\n') loop3_service.imagr(visA,cellsize='0.1asec',domultiscale=True,\ outname=visA+'_%02d'%iloop,dojoinchannels=True,channelsout=8,robust=0,\ fitsmask=fitsmask,dolocalrms=True,maxuvwm=cohlength) image_bdsf = '%s_%02d-MFS-image.fits' % (visA, iloop) pstr = '******* AMPLITUDE LOOP %d making mask %s_%02d-MFS-image.fits ************' % ( iloop, visA, iloop) loop3_service.loop3log(vis, pstr + '\n') img = bdsf.process_image(image_bdsf, atrous_do=True, thresh_isl=5.0) img.export_image(img_type='island_mask', outfile='%s_%02d-mask.fits' % (visA, iloop)) thisstat = measure_statistic(visA + '_%02d-MFS-image.fits' % iloop) pstr = '******* AMPLITUDE LOOP %d goodness stat %f ************' % ( iloop, thisstat) loop3_service.loop3log(vis, pstr + '\n') if iloop != ploop and thisstat - prevstat < 0.01: pstr = '****** EXITING AMPLITUDE CAL with diff %f *********' % ( thisstat - prevstat) loop3_service.loop3log(vis, pstr + '\n') break else: prevstat = thisstat loop3_service.imagr(visA,dopredict=True,fitsmask=fitsmask,autothreshold=2.5,dolocalrms=True,\ robust=0,outname=visA+'_%02d-MFS'%iloop) pstr = '******* AMPLITUDE LOOP %d making new cal file %s ************' % ( iloop, visA + '_%02d' % iloop) loop3_service.loop3log(vis, pstr + '\n') caltype, sol0 = strategy[iloop][0], float(strategy[iloop][1:]) coh,cohlength = selfcal(visA,model='MODEL',incol='DATA',outcol='CORRECTED_DATA',\ outcal_root=visA+'_%02d'%iloop,caltype=caltype,init_sol=sol0) pstr='******** END AMPLITUDE LOOP %d - coherence on %.1f km **********' % \ (iloop,cohlength/1000.) loop3_service.loop3log(vis, pstr + '\n') fitsmask = init_fitsmask if iloop == ploop else visA + '_%02d-mask.fits' % ( iloop - 1) loop3_service.imagr(visA,cellsize='0.1asec',domultiscale=True,\ outname=visA+'_final',dojoinchannels=True,channelsout=8,robust=0,\ fitsmask=fitsmask,dolocalrms=True) loop3_service.montage_plot(vis) pngfile, h5files = cleanup(vis) loop3_service.loop3log(vis, 'Output png file %s' % pngfile) print 'Output calibration tables', h5files return pngfile, h5files
def do_offsets(o): # o is the options file if o['mode'] != 'normal' and o['mode'] != 'test': raise NotImplementedError('Offsets called with mode ' + o['mode']) image_root = 'image_full_ampphase_di_m.NS' method = o['method'] report('Determining astrometric offsets with method ' + method + ' in mode ' + o['mode']) report('Merging downloaded catalogues') if os.path.isfile(method + '.fits'): warn('Merged file exists, reading from disk instead') data = Table.read(method + '.fits') else: if method == 'pslocal': data = Table.read(method + '/' + method + '.txt', format='ascii') data['RA'].name = 'ra' data['DEC'].name = 'dec' data.write(method + '.fits') else: kwargs = {} if 'panstarrs' in method: kwargs['rastr'] = 'ramean' kwargs['decstr'] = 'decmean' data = merge_cat(method, **kwargs) if o['mode'] == 'test': image_root += '_shift' method += '-test' report('Running PyBDSM on LOFAR image, please wait...') catfile = image_root + '.offset_cat.fits' gaulfile = catfile.replace('cat', 'gaul') if os.path.isfile(catfile): warn('Catalogue already exists, skipping pybdsf run') else: if o['mode'] == 'test': suffix = 'facetRestored' else: suffix = 'restored' pbimage = image_root + '.int.' + suffix + '.fits' nonpbimage = image_root + '.app.' + suffix + '.fits' img = bdsm.process_image(pbimage, detection_image=nonpbimage, thresh_isl=4.0, thresh_pix=5.0, rms_box=(150, 15), rms_map=True, mean_map='zero', ini_method='intensity', adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(60, 15), group_by_isl=False, group_tol=10.0, output_opts=True, output_all=True, atrous_do=False, flagging_opts=True, flag_maxsize_fwhm=0.5, advanced_opts=True, blank_limit=None) img.write_catalog(outfile=catfile, catalog_type='srl', format='fits', correct_proj='True') img.write_catalog(outfile=gaulfile, catalog_type='gaul', format='fits', correct_proj='True') lofar = Table.read(catfile) print len(lofar), 'LOFAR sources before filtering' filter = (lofar['E_RA'] * 3600.0) < 2.0 filter &= (lofar['E_DEC'] * 3600.0) < 2.0 filter &= (lofar['Maj'] * 3600.0) < 10 lofar = lofar[filter] print len(lofar), 'LOFAR sources after filtering' regfile = image_root + '.tessel.reg' cra, cdec = get_centpos() report('Set up structure') NDir = np.load("image_dirin_SSD_m.npy.ClusterCat.npy").shape[0] oo = Offsets(method, n=NDir, imroot=image_root, cellsize=o['cellsize'], fitmethod=o['fit']) report('Label table') lofar_l = oo.r.add_facet_labels(lofar) report('Finding offsets') oo.find_offsets(lofar_l, data) report('Fitting offsets') oo.fit_offsets() report('Making plots and saving output') #oo.plot_fits(method+'-fits.pdf') oo.save_fits() oo.plot_offsets() if 'test' not in o['mode']: oo.save(method + '-fit_state.pickle') report('Making astrometry error map, please wait') oo.make_astrometry_map('astromap.fits', 20) oo.offsets_to_facetshift('facet-offset.txt')
print 'Blanking the mosaic...' blank_mosaic('mosaic.fits',himsize) if args.do_lowres: print 'Making the low-resolution mosaic...' header,himsize=make_header(maxsep,mospointingname,pointingdict[mospointingname][1],pointingdict[mospointingname][2],4.5,20.0) mos_args.header=header mos_args.rootname='low' mos_args.do_lowres=True mos_args.astromap_blank=False # don't bother with low-res map #with open('low-mosaic-header.pickle','w') as f: # pickle.dump(header,f) make_mosaic(mos_args) print 'Blanking the mosaic...' blank_mosaic('low-mosaic.fits',himsize) print 'Now running PyBDSF to extract sources' catprefix='mosaic' img = bdsm.process_image('mosaic-blanked.fits', thresh_isl=4.0, thresh_pix=5.0, rms_box=(160,50), rms_map=True, mean_map='zero', ini_method='intensity', adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(60,15), group_by_isl=False, group_tol=10.0,output_opts=True, output_all=True, atrous_do=True,atrous_jmax=4, flagging_opts=True, flag_maxsize_fwhm=0.5,advanced_opts=True, blank_limit=None,frequency=restfrq) img.write_catalog(outfile=catprefix +'.cat.fits',catalog_type='srl',format='fits',correct_proj='True') img.export_image(outfile=catprefix +'.rms.fits',img_type='rms',img_format='fits',clobber=True) img.export_image(outfile=catprefix +'.resid.fits',img_type='gaus_resid',img_format='fits',clobber=True) img.export_image(outfile=catprefix +'.pybdsmmask.fits',img_type='island_mask',img_format='fits',clobber=True) img.write_catalog(outfile=catprefix +'.cat.reg',catalog_type='srl',format='ds9',correct_proj='True')
def find_sources(infile): img = bdsf.process_image(infile,thresh_pix=12.0,thresh_isl=5.0) img.write_catalog(format='ascii',catalog_type='gaul',clobber=True,incl_empty=True)
def do_sourcefinding(imagename): img = bdsf.process_image(imagename, adaptive_rms_box=True, advanced_opts=True,\ atrous_do=False, psf_vary_do=True, psf_snrcut=5.0, psf_snrcutstack=10.0,\ output_opts=True, output_all=True, opdir_overwrite='append', beam=(4.1666e-4, 4.1666e-4, 0),\ blank_limit=None, thresh='hard', thresh_isl=5.0, thresh_pix=7.0, psf_snrtop=0.30) return img
def main(image_name, mask_name, atrous_do=False, threshisl=0.0, threshpix=0.0, rmsbox=None, rmsbox_bright=(35, 7), iterate_threshold=False, adaptive_rmsbox=False, img_format='fits', threshold_format='float', trim_by=0.0, vertices_file=None, atrous_jmax=6, pad_to_size=None, skip_source_detection=False, region_file=None, nsig=1.0, reference_ra_deg=None, reference_dec_deg=None, cellsize_deg=0.000417, use_adaptive_threshold=False, make_blank_image=False, adaptive_thresh=150.0, exclude_cal_region=False, dilate=0): """ Make a clean mask and return clean threshold Parameters ---------- image_name : str Filename of input image from which mask will be made. If the image does not exist or make_blank_image is True, a template image with center at (reference_ra_deg, reference_dec_deg) will be made internally mask_name : str Filename of output mask image atrous_do : bool, optional Use wavelet module of PyBDSF? threshisl : float, optional Value of thresh_isl PyBDSF parameter threshpix : float, optional Value of thresh_pix PyBDSF parameter rmsbox : tuple of floats, optional Value of rms_box PyBDSF parameter rmsbox_bright : tuple of floats, optional Value of rms_box_bright PyBDSF parameter iterate_threshold : bool, optional If True, threshold will be lower in 20% steps until at least one island is found adaptive_rmsbox : tuple of floats, optional Value of adaptive_rms_box PyBDSF parameter img_format : str, optional Format of output mask image (one of 'fits' or 'casa') threshold_format : str, optional Format of output threshold (one of 'float' or 'str_with_units') trim_by : float, optional Fraction by which the perimeter of the output mask will be trimmed (zeroed) vertices_file : str, optional Filename of file with vertices (must be a pickle file containing a dictionary with the vertices in the 'vertices' entry) atrous_jmax : int, optional Value of atrous_jmax PyBDSF parameter pad_to_size : int, optional Pad output mask image to a size of pad_to_size x pad_to_size skip_source_detection : bool, optional If True, source detection is not run on the input image region_file : str, optional Filename of region file in CASA format to use as the mask. If update_user_mask is True, regions in region_file are unioned with ones found by the source finder nsig : float, optional Number of sigma of returned threshold value reference_ra_deg : float, optional RA for center of output mask image reference_dec_deg : float, optional Dec for center of output mask image cellsize_deg : float, optional Size of a pixel in degrees use_adaptive_threshold : bool, optional If True, use an adaptive threshold estimated from the negative values in the image make_blank_image : bool, optional If True, a blank template image is made. In this case, reference_ra_deg and reference_dec_deg must be specified adaptive_thresh : float, optional If adaptive_rmsbox is True, this value sets the threshold above which a source will use the small rms box exclude_cal_region : bool, optional If True, and a vertices_file is given, the calibrator region is also exclude from the output mask dilate : int, optional Number of dilation iterations for PyBDSF mask Returns ------- result : dict Dict with nsig-sigma rms threshold """ if rmsbox is not None and type(rmsbox) is str: rmsbox = eval(rmsbox) if type(rmsbox_bright) is str: rmsbox_bright = eval(rmsbox_bright) if pad_to_size is not None and type(pad_to_size) is str: pad_to_size = int(pad_to_size) if type(atrous_do) is str: if atrous_do.lower() == 'true': atrous_do = True threshisl = 4.0 # override user setting to ensure proper source fitting else: atrous_do = False if type(iterate_threshold) is str: if iterate_threshold.lower() == 'true': iterate_threshold = True else: iterate_threshold = False if type(adaptive_rmsbox) is str: if adaptive_rmsbox.lower() == 'true': adaptive_rmsbox = True else: adaptive_rmsbox = False if type(skip_source_detection) is str: if skip_source_detection.lower() == 'true': skip_source_detection = True else: skip_source_detection = False if type(use_adaptive_threshold) is str: if use_adaptive_threshold.lower() == 'true': use_adaptive_threshold = True else: use_adaptive_threshold = False if reference_ra_deg is not None and reference_dec_deg is not None: reference_ra_deg = float(reference_ra_deg) reference_dec_deg = float(reference_dec_deg) if type(make_blank_image) is str: if make_blank_image.lower() == 'true': make_blank_image = True else: make_blank_image = False if not os.path.exists(image_name): make_blank_image = True if type(exclude_cal_region) is str: if exclude_cal_region.lower() == 'true': exclude_cal_region = True else: exclude_cal_region = False dilate = int(dilate) if make_blank_image: print('Making empty template image...') if not skip_source_detection: print('ERROR: Source detection cannot be done on an empty image') sys.exit(1) if reference_ra_deg is not None and reference_dec_deg is not None: image_name = mask_name + '.tmp' make_template_image(image_name, reference_ra_deg, reference_dec_deg, cellsize_deg=float(cellsize_deg)) else: print('ERROR: a reference position must be given to make an empty template image') sys.exit(1) trim_by = float(trim_by) atrous_jmax = int(atrous_jmax) threshpix = float(threshpix) threshisl = float(threshisl) nsig = float(nsig) adaptive_thresh = float(adaptive_thresh) threshold = 0.0 nisl = 0 if not skip_source_detection: if vertices_file is not None: # Modify the input image to blank the regions outside of the polygon blank_image.main(image_name, vertices_file, image_name+'.blanked', blank_value='nan') image_name += '.blanked' if use_adaptive_threshold: # Get an estimate of the rms img = bdsf.process_image(image_name, mean_map='zero', rms_box=rmsbox, thresh_pix=threshpix, thresh_isl=threshisl, atrous_do=atrous_do, thresh='hard', adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright, rms_map=True, quiet=True, atrous_jmax=atrous_jmax, stop_at='isl') # Find min and max pixels max_neg_val = abs(np.min(img.ch0_arr)) max_neg_pos = np.where(img.ch0_arr == np.min(img.ch0_arr)) max_pos_val = abs(np.max(img.ch0_arr)) max_pos_pos = np.where(img.ch0_arr == np.max(img.ch0_arr)) # Estimate new thresh_isl from min pixel value's sigma, but don't let # it get higher than 1/2 of the peak's sigma threshisl_neg = 2.0 * max_neg_val / img.rms_arr[max_neg_pos][0] max_sigma = max_pos_val / img.rms_arr[max_pos_pos][0] if threshisl_neg > max_sigma / 2.0: threshisl_neg = max_sigma / 2.0 # Use the new threshold only if it is larger than the user-specified one if threshisl_neg > threshisl: threshisl = threshisl_neg if not atrous_do: stop_at = 'isl' else: stop_at = None if iterate_threshold: # Start with given threshold and lower it until we get at least one island nisl = 0 while nisl == 0: img = bdsf.process_image(image_name, mean_map='zero', rms_box=rmsbox, thresh_pix=threshpix, thresh_isl=threshisl, atrous_do=atrous_do, thresh='hard', adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright, rms_map=True, quiet=True, atrous_jmax=atrous_jmax, stop_at=stop_at) nisl = img.nisl threshpix /= 1.2 threshisl /= 1.2 if threshpix < 5.0: break else: img = bdsf.process_image(image_name, mean_map='zero', rms_box=rmsbox, thresh_pix=threshpix, thresh_isl=threshisl, atrous_do=atrous_do, thresh='hard', adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright, rms_map=True, quiet=True, atrous_jmax=atrous_jmax, stop_at=stop_at) nisl = img.nisl if nisl == 0: if region_file is None or region_file == '[]': print('No islands found. Clean mask cannot be made.') return {'threshold_5sig': 'None'} else: # Continue on and use user-supplied region file threshold = nsig * img.clipped_rms else: # Write out the mask img.export_image(img_type='island_mask', mask_dilation=dilate, outfile=mask_name, img_format=img_format, clobber=True) # Check if there are large islands present (indicating that multi-scale # clean is needed) has_large_isl = False for isl in img.islands: if isl.size_active > 100: # Assuming normal sampling, a size of 100 pixels would imply # a source of ~ 10 beams has_large_isl = True if (vertices_file is not None or trim_by > 0 or pad_to_size is not None or (region_file is not None and region_file != '[]') or skip_source_detection): # Alter the mask in various ways if skip_source_detection or nisl == 0: # Read the image mask_im = pim.image(image_name) else: # Read the PyBDSF mask mask_im = pim.image(mask_name) data = mask_im.getdata() coordsys = mask_im.coordinates() if reference_ra_deg is not None and reference_dec_deg is not None: values = coordsys.get_referencevalue() values[2][0] = reference_dec_deg/180.0*np.pi values[2][1] = reference_ra_deg/180.0*np.pi coordsys.set_referencevalue(values) imshape = mask_im.shape() del(mask_im) if pad_to_size is not None: imsize = pad_to_size coordsys['direction'].set_referencepixel([imsize/2, imsize/2]) pixmin = (imsize - imshape[2]) / 2 if pixmin < 0: print("The padded size must be larger than the original size.") sys.exit(1) pixmax = pixmin + imshape[2] data_pad = np.zeros((1, 1, imsize, imsize), dtype=np.float32) data_pad[0, 0, pixmin:pixmax, pixmin:pixmax] = data[0, 0] new_mask = pim.image('', shape=(1, 1, imsize, imsize), coordsys=coordsys) new_mask.putdata(data_pad) else: new_mask = pim.image('', shape=imshape, coordsys=coordsys) new_mask.putdata(data) data = new_mask.getdata() if skip_source_detection or nisl == 0: if region_file is not None and region_file != '[]': # Unmask all pixels. We will fill the masked regions # below data[:] = 0 else: # Mask all pixels data[:] = 1 if region_file is not None and region_file != '[]': # Merge the CASA regions with the mask casa_polys = read_casa_polys(region_file.strip('[]"'), new_mask) for poly in casa_polys: # Find unmasked regions unmasked_ind = np.where(data[0, 0] == 0) # Find distance to nearest poly edge and mask those that # are inside the casa region (dist > 0) dist = poly.is_inside(unmasked_ind[0], unmasked_ind[1]) inside_ind = np.where(dist > 0.0) if len(inside_ind[0]) > 0: data[0, 0, unmasked_ind[0][inside_ind], unmasked_ind[1][inside_ind]] = 1 if vertices_file is not None: # Modify the clean mask to exclude regions outside of the polygon vertices = read_vertices(vertices_file) poly = vertices_to_poly(vertices, new_mask) if exclude_cal_region: cal_vertices = read_vertices(vertices_file, cal_only=True) cal_poly = vertices_to_poly(cal_vertices, new_mask) # Find masked regions masked_ind = np.where(data[0, 0]) # Find distance to nearest poly edge and unmask those that # are outside the facet (dist < 0) and inside the calibrator region # (cal_dist > 0) dist = poly.is_inside(masked_ind[0], masked_ind[1]) outside_ind = np.where(dist < 0.0) if len(outside_ind[0]) > 0: data[0, 0, masked_ind[0][outside_ind], masked_ind[1][outside_ind]] = 0 if exclude_cal_region: masked_ind = np.where(data[0, 0]) cal_dist = cal_poly.is_inside(masked_ind[0], masked_ind[1]) inside_ind = np.where(cal_dist > 0.0) if len(inside_ind[0]) > 0: data[0, 0, masked_ind[0][inside_ind], masked_ind[1][inside_ind]] = 0 if trim_by > 0.0: sh = np.shape(data) margin = int(sh[2] * trim_by / 2.0 ) data[0, 0, 0:sh[2], 0:margin] = 0 data[0, 0, 0:margin, 0:sh[3]] = 0 data[0, 0, 0:sh[2], sh[3]-margin:sh[3]] = 0 data[0, 0, sh[2]-margin:sh[2], 0:sh[3]] = 0 # Save changes new_mask.putdata(data) if img_format == 'fits': new_mask.tofits(mask_name, overwrite=True) elif img_format == 'casa': new_mask.saveas(mask_name, overwrite=True) else: print('Output image format "{}" not understood.'.format(img_format)) sys.exit(1) if not skip_source_detection: if threshold_format == 'float': return {'threshold_5sig': nsig * img.clipped_rms, 'multiscale': has_large_isl} elif threshold_format == 'str_with_units': # This is done to get around the need for quotes around strings in casapy scripts # 'casastr/' is removed by the generic pipeline return {'threshold_5sig': 'casastr/{0}Jy'.format(nsig * img.clipped_rms), 'multiscale': has_large_isl} else: return {'threshold_5sig': '0.0'}