Esempio n. 1
0
def main(inmodel1, inmodel2, outmodel, match_by='name', radius=0.0, keep='all'):
    """
    Creates mosaic

    Parameters
    ----------
    inmodel1 : str
        Filename of input model 1
    inmodel2 : str
        Filename of input model 2
    outmodel : str
        Filename of output model
    match_by : str, optional
        The match_by parameter of LSMTool
    radius : float, optional
        Radius in degrees for cross match
    keep : str, optional
        The keep parameter of LSMTool

    """
    s1 = lsmtool.load(inmodel1)
    s2 = lsmtool.load(inmodel2)

    s1.concatenate(s2, matchBy=match_by, radius=float(radius), keep=keep,
        inheritPatches=True)
    s1.group('every')
    s1.write(fileName=outmodel, clobber=True)
Esempio n. 2
0
def main(model1, model2, skymodel):
    """
    Combines makesourcedb sky models

    Parameters
    ----------
    model1 : str
        Filename of the input makesourcedb sky model 1
    model2 : str
        Filename of the input makesourcedb sky model 2
    skymodel : str
        Filename of the output makesourcedb sky model

    """
    try:
        s1 = lsmtool.load(model1)
    except:
        # If first sky model is empty or cannot be loaded, just copy second one
        # to output file
        os.system('cp -f {0} {1}'.format(model2, skymodel))
        return

    # Now try to load second sky model and combine with first one
    try:
        s2 = lsmtool.load(model2)

        # Combine sky models, keeping all sources
        s1.ungroup()
        s2.ungroup()
        s1.concatenate(s2, keep='all')
    except:
        # If second sky model is empty or cannot be loaded, just save s1 to output
        pass

    s1.write(skymodel, clobber=True)
Esempio n. 3
0
def main(inmodel1,
         inmodel2,
         outmodel,
         match_by='name',
         radius=0.0,
         keep='all'):
    """
    Creates mosaic

    Parameters
    ----------
    inmodel1 : str
        Filename of input model 1
    inmodel2 : str
        Filename of input model 2
    outmodel : str
        Filename of output model
    match_by : str, optional
        The match_by parameter of LSMTool
    radius : float, optional
        Radius in degrees for cross match
    keep : str, optional
        The keep parameter of LSMTool

    """
    s1 = lsmtool.load(inmodel1)
    s2 = lsmtool.load(inmodel2)

    s1.concatenate(s2,
                   matchBy=match_by,
                   radius=float(radius),
                   keep=keep,
                   inheritPatches=True)
    s1.group('every')
    s1.write(fileName=outmodel, clobber=True)
Esempio n. 4
0
    def selectCC(self, checkBeam=True, keepInBeam=True, maskname=None):
        """
        remove cc from a skymodel according to masks
        checkBeam: remove according to beam (see keepInBeam)
        keepInBeam: if beamReg is present and is True: remove sources outside beam
                    if beamReg is present and is False: remove source inside beam
        maskname: a possible mask, otherwise try to use standard
        """
        if maskname is None: maskname = self.maskname
        if not os.path.exists(maskname):
            raise("Missing mask in selectCC: %s." % maskname)

        if checkBeam:
            if self.beamReg is None:
                raise('Missing beam in selectCC.')
            logger.info('Predict (apply beam reg %s)...' % self.beamReg)
            blank_image_reg(maskname, self.beamReg, inverse=keepInBeam, blankval=0) # if keep_in_beam set to 0 everything outside beam.reg

        # apply mask
        logger.info('%s: Apply mask (%s) on skymodel...' % (self.imagename,maskname))
        lsm = lsmtool.load(self.skymodel)
        lsm.select('%s == True' % maskname)
        lsm.group('single') # group to 1 patch
        lsm.write(self.skymodel_cut, format = 'makesourcedb', clobber=True)
        del lsm

        # convert from txt to blob
        logger.info('%s: Make skydb...' % self.imagename)
        lib_util.check_rm(self.skydb)
        os.system('makesourcedb outtype="blob" format="<" in="'+self.skymodel_cut+'" out="'+self.skydb+'"')
Esempio n. 5
0
def main(fullskymodel,
         outmodel,
         cal_only=False,
         vertices=None,
         facet_ra=0.0,
         facet_dec=0.0,
         cal_radius_deg=0.0):
    print 'Running {0} on input data {1}'.format(__file__, str(fullskymodel))

    s = lsmtool.load(fullskymodel)

    if cal_only:
        # Get calibrator model
        dist = s.getDistance(facet_ra, facet_dec)
        s.select(dist < cal_radius_deg)
    else:
        # Get all facet sources
        x, y, midRA, midDec = s._getXY()
        xv, yv = radec2xy(vertices[0], vertices[1], midRA, midDec)
        xyvertices = array([[xp, yp] for xp, yp in zip(xv, yv)])
        bbPath = mplPath.Path(xyvertices)
        inside = zeros(len(s), dtype=bool)
        for i in range(len(s)):
            inside[i] = bbPath.contains_point((x[i], y[i]))
        s.select(inside, force=True)

    if len(s) == 0:
        print('No sources found for this facet')
    else:
        s.write(outmodel, clobber=True)
Esempio n. 6
0
def make_initial_skymodel(band):
    """
    Makes the initial skymodel used to adjust facet edges

    Parameters
    ----------
    band : Band object
        Band to use for sky model generation

    Returns
    -------
    s : LSMTool Skymodel object
        Resulting sky model

    """
    import lsmtool

    # Set LSMTool logging level
    lsmtool._logging.setLevel('debug')

    # Load sky model
    s = lsmtool.load(band.skymodel_dirindep)

    # Group clean components by thresholding after convolving model with
    # 1-arcmin beam
    s.group('threshold', FWHM='60.0 arcsec', root='facet')
    s.remove('Patch = patch_*', force=True) # Remove sources that did not threshold
    if len(s) == 0:
        log.critical("No sources found through thresholding.")
        sys.exit(1)
    log.info('Found {0} sources through thresholding'.format(
        len(s.getPatchNames())))

    return s
Esempio n. 7
0
    def selectCC(self, keepInBeam=True):
        """
        remove cc from a skymodel according to masks
        keepInBeam: if beamReg is present and is True: remove sources outside beam
                    if beamReg is present and is False: remove source inside beam
        """
        self.makeMask()

        if self.facetReg is not None:
            logger.info('Predict (apply facet reg %s)...' % self.facetReg)
            blank_image_reg(self.maskname, self.facetReg, inverse=True, blankval=0) # set to 0 pixels outside facet mask

        if self.beamReg is not None:
            logger.info('Predict (apply beam reg %s)...' % self.beamReg)
            blank_image_reg(self.maskname, self.beamReg, inverse=keepInBeam, blankval=0) # if keep_in_beam set to 0 everything outside beam.reg

        # apply mask
        logger.info('%s: Apply mask on skymodel...' % self.imagename)
        lsm = lsmtool.load(self.skymodel)
        lsm.select('%s == True' % self.maskname)
        lsm.group('single') # group to 1 patch
        lsm.write(self.skymodel_cut, format = 'makesourcedb', clobber=True)
        del lsm

        # convert from txt to blob
        logger.info('%s: Make skydb...' % self.imagename)
        lib_util.check_rm(self.skydb)
        os.system('makesourcedb outtype="blob" format="<" in="'+self.skymodel_cut+'" out="'+self.skydb+'"')
def read_skymodel(skymodel, sagecalsky, sagecalcluster, admm_rho='base.rho'):
    outsky = open(sagecalsky, 'w+')
    outcluster = open(sagecalcluster, 'w+')
    outrho = open(admm_rho, 'w+')
    outsky.write('## LSM file\n')
    outsky.write(
        "### Name  | RA (hr,min,sec) | DEC (deg,min,sec) | I | Q | U |  V | SI0 | SI1 | SI2 | RM | eX | eY | eP | freq0\n"
    )
    outcluster.write('### Cluster file\n')
    s = lsmtool.load(skymodel)
    patches = s.getPatchNames()
    cluster_id = 2  # cluster 1 is the target
    for patch in patches:
        s = lsmtool.load(skymodel)
        s.select('Patch == ' + patch)
        t = s.table
        ra = ra_to_rad(np.array(t['Ra']))
        dec = dec_to_rad(np.array(t['Dec']))
        stype = [x.encode('ascii') for x in np.array(t['Type'])]
        f0 = np.array(t['ReferenceFrequency'])
        sI = np.array(t['I'])
        SpecI = np.array(t['SpectralIndex'][:, 0])
        major = asec_to_rad(np.array(t['MajorAxis']))
        minor = asec_to_rad(np.array(t['MinorAxis']))
        pa = math.pi / 2 - (math.pi - dec_to_rad(np.array(t['Orientation'])))
        outcluster.write(str(cluster_id) + ' 1')
        outrho.write(str(cluster_id) + ' 1 1.0\n')
        for ci in range(ra.size):
            hh, mm, ss = radToRA(ra[ci])
            dd, dmm, dss = radToDec(dec[ci])
            if stype[ci].decode('ascii') == 'GAUSSIAN':
                name = 'G' + patch + str(ci)
            else:
                name = 'P' + patch + str(ci)
            outsky.write(name + ' ' + str(hh) + ' ' + str(mm) + ' ' + str(ss) +
                         ' ' + str(dd) + ' ' + str(dmm) + ' ' + str(dss) +
                         ' ' + str(sI[ci]) + ' 0 0 0' + ' ' + str(SpecI[ci]) +
                         ' 0 0 0' + ' ' + str(0.5 * major[ci]) + ' ' +
                         str(0.5 * minor[ci]) + ' ' + str(pa[ci]) + ' ' +
                         str(f0[ci]) + '\n')
            outcluster.write(' ' + name)
        outcluster.write('\n')
        cluster_id += 1
    outsky.close()
    outcluster.close()
    outrho.close()
Esempio n. 9
0
def test_compare():
    print('Compare to concat.sky')
    if os.path.exists('tests/flux_ratio_vs_distance.sky'):
        os.remove('tests/flux_ratio_vs_distance.sky')
    c = lsmtool.load('tests/concat.sky')
    c.ungroup()
    c.select('I > 5.0 Jy')
    s.ungroup()
    s.compare(c, outDir='tests/')
    assert os.path.exists('tests/flux_ratio_vs_distance.pdf')
Esempio n. 10
0
def test_compare():
    print('Compare to concat.sky')
    if os.path.exists('tests/flux_ratio_vs_distance.sky'):
        os.remove('tests/flux_ratio_vs_distance.sky')
    c = lsmtool.load('tests/concat.sky')
    c.ungroup()
    c.select('I > 5.0 Jy')
    s.ungroup()
    s.compare(c, outDir='tests/')
    assert os.path.exists('tests/flux_ratio_vs_distance.pdf')
Esempio n. 11
0
def ft_model_cc(mss,
                imagename,
                c,
                user_mask=None,
                keep_in_beam=True,
                model_column='MODEL_DATA'):
    """
    skymodel : cc-list made by wsclean
    keep_in_beam : if True remove everything outside primary beam, otherwise everything inside
    """
    logger.info('Predict with CC...')
    maskname = imagename + '-mask.fits'
    skymodel = imagename + '-sources.txt'
    skymodel_cut = imagename + '-sources-cut.txt'
    skydb = imagename + '-sources.skydb'

    # prepare mask
    if not os.path.exists(maskname):
        logger.info('Predict (make mask)...')
        make_mask(image_name=imagename + '-MFS-image.fits',
                  mask_name=maskname,
                  threshisl=5,
                  atrous_do=True)
    if user_mask is not None:
        blank_image_reg(maskname, user_mask, inverse=False,
                        blankval=1)  # set to 1 pixels into user_mask
    blank_image_reg(
        maskname, 'self/beam.reg', inverse=keep_in_beam,
        blankval=0)  # if keep_in_beam set to 0 everything outside beam.reg

    # apply mask
    logger.info('Predict (apply mask)...')
    lsm = lsmtool.load(skymodel)
    lsm.select('%s == True' % maskname)
    fluxes = lsm.getColValues('I')
    #lsm.remove(np.abs(fluxes) < 5e-4) # TEST
    lsm.write(skymodel_cut, format='makesourcedb', clobber=True)
    del lsm

    # convert to skydb
    logger.info('Predict (makesourcedb)...')
    check_rm(skydb)
    s.add('makesourcedb outtype="blob" format="<" in="' + skymodel_cut +
          '" out="' + skydb + '"',
          log='makesourcedb-c' + str(c) + '.log',
          cmd_type='general')
    s.run(check=True)

    # predict
    logger.info('Predict (ft)...')
    for ms in mss:
        s.add('NDPPP '+parset_dir+'/NDPPP-predict.parset msin='+ms+' msout.datacolumn='+model_column+' pre.usebeammodel=false pre.sourcedb='+skydb, \
                log=ms+'_pre-c'+str(c)+'.log', cmd_type='NDPPP')
    s.run(check=True)
Esempio n. 12
0
def check_skymodel(skymodel, ra, dec, max_separation_arcmin=1.0):
    """
    Searches for an appropriate sky model
    """
    s = lsmtool.load(skymodel)
    dist_deg = s.getDistance(ra, dec)
    if any(dist_deg * 60.0 < max_separation_arcmin):
        patch_position = int(
            numpy.where(dist_deg * 60 < max_separation_arcmin)[0][0])
        patch_name = s.getPatchNames()[patch_position]
        return (True, patch_name)
    else:
        return (False, '')
Esempio n. 13
0
def make_initial_skymodel(band, max_radius_deg=None):
    """
    Makes the initial skymodel used to adjust facet edges

    Parameters
    ----------
    band : Band object
        Band to use for sky model generation

    Returns
    -------
    s : LSMTool Skymodel object
        Resulting sky model
    max_radius_deg : float, optional
        Maximum radius in degrees from the phase center within which to include
        sources. If None, it is set to the FWHM (i.e., a diameter of 2 * FWHM)

    """
    import lsmtool

    # Set LSMTool logging level
    lsmtool._logging.setLevel('debug')

    # Load sky model
    s = lsmtool.load(band.skymodel_dirindep)

    # Group clean components by thresholding after convolving model with
    # 1-arcmin beam
    s.group('threshold', FWHM='60.0 arcsec', root='facet')
    s.remove('Patch = patch_*', force=True) # Remove sources that did not threshold
    if len(s) == 0:
        log.critical("No sources found through thresholding.")
        sys.exit(1)
    log.info('Found {0} sources through thresholding'.format(
        len(s.getPatchNames())))

    # Filter out sources that lie outside of maximum specific radius from phase
    # center
    if not hasattr(band, 'fwhm_deg'):
        band.set_image_sizes()
    if max_radius_deg is None:
        max_radius_deg = band.fwhm_deg # means a diameter of 2 * FWHM

    log.info('Removing sources beyond a radius of {0} degrees (corresponding to '
        'a diameter of {1} * FWHM of the primary beam at {2} MHz)...'.format(
        max_radius_deg, round(2.0*max_radius_deg/band.fwhm_deg, 1), band.freq/1e6))

    dist = s.getDistance(band.ra, band.dec, byPatch=True)
    s.remove(dist > max_radius_deg, aggregate=True)

    return s
Esempio n. 14
0
def cut_skymodel(skymodel_in, skymodel_out, d, do_skydb=True, do_regions=False):
    """
    Load full skymodel and extract sources in the square around the calibrator of the given size
    """
    lsm = lsmtool.load(skymodel_in)
    # select all sources within a sqare of patch size
    lsm.select('Ra > %f' % (d.position[0]-(d.size/2)/np.cos(d.position[1]* np.pi / 180.)))
    lsm.select('Ra < %f' % (d.position[0]+(d.size/2)/np.cos(d.position[1]* np.pi / 180.)))
    lsm.select('Dec > %f' % (d.position[1]-d.size/2))
    lsm.select('Dec < %f' % (d.position[1]+d.size/2))
    if do_regions: lsm.write('ddcal/masks/regions-c%02i/%s.reg' % (cmaj,d.name), format='ds9', clobber=True)
    lsm.write(dir_skymodel, format='makesourcedb', clobber=True)
    lib_util.check_rm(dir_skydb)
    s.add('makesourcedb outtype="blob" format="<" in="%s" out="%s"' % (dir_skymodel, dir_skydb), log='makesourcedb_cl.log', commandType='general' )
    s.run(check=True)
Esempio n. 15
0
def main(model, skymodel):
    """
    Groups a sky model into a single patch

    Parameters
    ----------
    model : str
        Filename of the input makesourcedb sky model
    skymodel : str
        Filename of the output makesourcedb sky model

    """
    s = lsmtool.load(model)
    s.group('single')

    s.write(skymodel, clobber=True)
Esempio n. 16
0
    def select_cc(self):
        """
        remove cc from a skymodel according to masks
        """
        self.make_mask()

        if self.region_facet is not None:
            logger.info('Predict (apply facet mask %s)...' % self.region_facet)
            blank_image_reg(self.maskname, self.region_facet, inverse=True, blankval=0) # set to 0 pixels outside facet mask

        # apply mask
        logger.info('%s: Apply mask on skymodel...' % self.imagename)
        lsm = lsmtool.load(self.skymodel)
        lsm.select('%s == True' % self.maskname)
        lsm.write(self.skymodel_cut, format='makesourcedb', clobber=True)
        del lsm
Esempio n. 17
0
def main(model, skymodel):
    """
    Groups a sky model into a single patch

    Parameters
    ----------
    model : str
        Filename of the input makesourcedb sky model
    skymodel : str
        Filename of the output makesourcedb sky model

    """
    s = lsmtool.load(model)
    s.group('single')

    s.write(skymodel, clobber=True)
Esempio n. 18
0
 def load_skymodel(self):
     """
     Loads the sky model
     """
     # Set logging level to suppress confusing output from lsmtool
     old_level = logger.root.getEffectiveLevel()
     logger.root.setLevel('WARNING')
     skymodel = lsmtool.load(self.input_skymodel_filename)
     if not skymodel.hasPatches:
         logger.info(
             'No patches present in skymodel. Assigning every source an individual patch.'
         )
         skymodel.group('every')
         skymodel.setPatchPositions(method='mid')
     logger.root.setLevel(old_level)
     self.skymodel = skymodel
Esempio n. 19
0
    def make_catalogue(self):
        """
        Create catalogue for this image.
        """
        import bdsf
        from astropy.table import Table

        img_cat = self.imagefile + '.cat'
        if not os.path.exists(img_cat):
            bdsf_img = bdsf.process_image(self.imagefile, rms_box=(100,30), \
                                          thresh_pix=5, thresh_isl=3, atrous_do=False, \
                                          adaptive_rms_box=True, adaptive_thresh=100, rms_box_bright=(30,10), quiet=True)
            bdsf_img.write_catalog(outfile=img_cat,
                                   catalog_type='srl',
                                   format='fits',
                                   clobber=True)
            bdsf_img.write_catalog(outfile=img_cat.replace(
                '.cat', '.skymodel'),
                                   catalog_type='gaul',
                                   format='bbs',
                                   bbs_patches='source',
                                   clobber=True,
                                   srcroot='src')
        else:
            logging.warning('%s already exists, using it.' % img_cat)

        cat = Table.read(img_cat)
        # remove extended sources
        extended_src = (cat['Peak_flux'] /
                        cat['Total_flux']) < 0.1  # ~extended source
        extended_src[cat['S_Code'] == 'M'] = True  # multiple-gaussian source
        extended_src[cat['S_Code'] ==
                     'C'] = True  # one gaussian + other sources island
        # remove same sources from skymodel
        cat_lsm = lsm.load(img_cat.replace('.cat', '.skymodel'))
        for srcid in cat[extended_src]['Source_id']:
            cat_lsm.remove(f'Patch == src_patch_s{srcid}')
        cat.remove_rows(np.argwhere(extended_src))
        self.cat = cat
        self.cat_lsm = cat_lsm
        logging.debug(
            '%s: Number of sources detected: %i; removed %i extended sources.'
            % (self.imagefile, len(self.cat), sum(extended_src)))
Esempio n. 20
0
    def find_peel_skymodel(self):
        """
        Searches for an appropriate sky model for peeling
        """
        if self.peel_skymodel is not None:
            return

        max_separation_arcmin = 1.0
        factor_lib_dir = os.path.dirname(os.path.abspath(__file__))
        skymodel_dir = os.path.join(os.path.split(factor_lib_dir)[0], 'skymodels')
        skymodels = glob.glob(os.path.join(skymodel_dir, '*.skymodel'))
        for skymodel in skymodels:
            try:
                s = lsmtool.load(skymodel)
                dist_deg = s.getDistance(self.ra, self.dec)
                if any(dist_deg*60.0 < max_separation_arcmin):
                    self.peel_skymodel = skymodel
                    break
            except IOError:
                pass
Esempio n. 21
0
def main(fullskymodel, outmodel, vertices_file, cal_only=False):
    """
    Makes a makesourcedb sky model for components inside input polygon

    Parameters
    ----------
    fullskymodel : str
        Filename of makesourcedb sky model file containing the full-field model
    outmodel : str
        Filename of output sky model
    vertices_file : str
        Filename of pickled file with direction vertices that define polygon
    cal_only : bool, optional
        If True, only components wihtin cal_radius_deg of (facet_ra, facet_dec)
        are selected.

    """
    if type(cal_only) is str:
        if cal_only.lower() == 'true':
            cal_only = True
        else:
            cal_only = False

    s = lsmtool.load(fullskymodel)
    vertices = read_vertices(vertices_file, cal_only=cal_only)

    # Select sources inside poly defined by vertices
    x, y, midRA, midDec = s._getXY()
    xv, yv = radec2xy(vertices[0], vertices[1], midRA, midDec)
    xyvertices = array([[xp, yp] for xp, yp in zip(xv, yv)])
    bbPath = mplPath.Path(xyvertices)
    inside = zeros(len(s), dtype=bool)
    for i in range(len(s)):
        inside[i] = bbPath.contains_point((x[i], y[i]))
    s.select(inside, force=True)

    if len(s) == 0:
        print('No sources found for this facet')
        os.system('touch {0}'.format(outmodel))
    else:
        s.write(outmodel, clobber=True)
def main(fullskymodel, outmodel, cal_only=False, vertices=None, facet_ra=0.0, facet_dec=0.0, cal_radius_deg=0.0):
    print 'Running {0} on input data {1}'.format(__file__, str(fullskymodel))

    s = lsmtool.load(fullskymodel)

    if cal_only:
        # Get calibrator model
        dist = s.getDistance(facet_ra, facet_dec)
        s.select(dist < cal_radius_deg)
    else:
        # Get all facet sources
        x, y, midRA, midDec = s._getXY()
        xv, yv = radec2xy(vertices[0], vertices[1], midRA, midDec)
        xyvertices = array([[xp, yp] for xp, yp in zip(xv, yv)])
        bbPath = mplPath.Path(xyvertices)
        inside = zeros(len(s), dtype=bool)
        for i in range(len(s)):
            inside[i] = bbPath.contains_point((x[i], y[i]))
        s.select(inside, force=True)

    if len(s) == 0:
        print('No sources found for this facet')
    else:
        s.write(outmodel, clobber=True)
Esempio n. 23
0
def main(model_root,
         ms_file,
         skymodel,
         fits_mask=None,
         min_peak_flux_jy=0.0001,
         max_residual_jy=0.0):
    """
    Make a makesourcedb sky model for input MS from WSClean fits model images

    Parameters
    ----------
    model_root : str
        Root name of WSClean polynomial model sky model
    ms_file : str
        Filename of MS for which sky model is to be made. Can be a list of files
        (e.g., '[ms1,ms2,...]', in which case they should all have the same
        frequency
    skymodel : str
        Filename of the output makesourcedb sky model
    fits_mask : str, optional
        Filename of fits mask
    min_peak_flux_jy : float, optional
        Minimum absolute value of flux in Jy of a source in lowest-frequency model image
        to include in output model
    max_residual_jy : float, optional
        Maximum acceptible total residual absolute flux in Jy

    """
    min_peak_flux_jy = float(min_peak_flux_jy)
    max_residual_jy = float(max_residual_jy)

    if type(fits_mask) is str:
        if fits_mask.lower() == 'none':
            fits_mask = None

    # Read MS file and get the frequency info
    if '[' in ms_file and ']' in ms_file:
        files = ms_file.strip('[]').split(',')
        files = [f.strip() for f in files]
        ms_file = files[0]
    sw = pt.table(ms_file + '::SPECTRAL_WINDOW', ack=False)
    ms_freq = sw.col('REF_FREQUENCY')[0]
    sw.close()

    # Read in sky model
    polymodel = model_root + '-sources.txt'
    s = lsmtool.load(polymodel)
    ref_freq = float(s.getColValues('ReferenceFrequency')[0])  # Hz

    # Find model images and read in frequencies
    fits_models = glob.glob(model_root + '-00*-model.fits')
    if len(fits_models) == 0:
        # No channels images found, so look for non-MFS images
        fits_models = glob.glob(model_root + '-model.fits')
    if len(fits_models) == 0:
        print('ERROR: no model images found')
        sys.exit(1)
    freqs = []
    for f in fits_models:
        # Get the frequency info
        hdr = fits.getheader(f, 0, ignore_missing_end=True)
        freqs.append(hdr['CRVAL3'])  # Hz

    # Determine nearest model frequency to MS frequency. We will use this to
    # get the fluxes (to match the frequency blocks used during imaging and
    # calibration)
    sky_freq = min(freqs, key=lambda x: abs(x - ms_freq))

    # Check if fits mask is empty
    if fits_mask is not None:
        if fits_mask.lower() == 'empty':
            # Handle case in which no sources were found during masking
            s.remove(np.array(range(len(s))))
            mask = None
        else:
            mask = fits.getdata(fits_mask, 0, ignore_missing_end=True)
            hdr = fits.getheader(fits_mask, 0, ignore_missing_end=True)
            w = wcs.WCS(hdr)
    else:
        mask = None

    # Discard components not in the mask, if given
    if mask is not None:
        pix = w.wcs_world2pix(
            np.array([
                s.getColValues('RA'),
                s.getColValues('Dec'), [0] * len(s), [0] * len(s)
            ]).T, 0)
        not_in_mask = []
        for i, p in enumerate(pix):
            if mask[0, 0, int(round(p[1])), int(round(p[0]))] < 1:
                not_in_mask.append(i)
        s.remove(np.array(not_in_mask))

    # Set fluxes and ref frequency
    specterms = s.getColValues('SpectralIndex')
    stokesI = s.getColValues('I')
    fluxes = []
    for i in range(len(s)):
        # Find flux as follows:
        #     flux(nu) = stokesI + term0 (nu/refnu - 1) + term1 (nu/refnu - 1)^2 + ...,
        # where nu = sky_freq and refnu = ref_freq
        polyterms = specterms[i].tolist()
        polyterms.insert(0, stokesI[i])
        fluxes.append(polyval(sky_freq / ref_freq - 1.0, polyterms))
    s.setColValues('I', fluxes)
    s.setColValues('ReferenceFrequency', [ms_freq] * len(s))

    # Remove spectral index values
    s.table.remove_column('SpectralIndex')
    s.table.remove_column('LogarithmicSI')

    # Write the new sky model
    s.write(fileName=skymodel, clobber=True)
Esempio n. 24
0
# set image size
imgsizepix = int(2.1 * MSs.getListObj()[0].getFWHM(freq='mid') * 3600 / 10.)
if imgsizepix % 2 != 0: imgsizepix += 1  # prevent odd img sizes

#################################################################
# Get online model
if sourcedb == '':
    if not os.path.exists('tgts.skydb'):
        fwhm = MSs.getListObj()[0].getFWHM(freq='min')
        radeg = phasecentre[0]
        decdeg = phasecentre[1]
        # get model the size of the image (radius=fwhm/2)
        os.system(
            'wget -O tgts.skymodel "https://lcs165.lofar.eu/cgi-bin/gsmv1.cgi?coord=%f,%f&radius=%f&unit=deg"'
            % (radeg, decdeg, fwhm / 2.))  # ASTRON
        lsm = lsmtool.load('tgts.skymodel')  #, beamMS=MSs.getListObj()[0])
        lsm.remove('I<1')
        lsm.write('tgts.skymodel', clobber=True)
        os.system(
            'makesourcedb outtype="blob" format="<" in=tgts.skymodel out=tgts.skydb'
        )
        apparent = False

    sourcedb = 'tgts.skydb'

#################################################################################################
# Add model to MODEL_DATA
# copy sourcedb into each MS to prevent concurrent access from multiprocessing to the sourcedb
sourcedb_basename = sourcedb.split('/')[-1]
for MS in MSs.getListStr():
    lib_util.check_rm(MS + '/' + sourcedb_basename)
Esempio n. 25
0
        #if c == 0:
        #    clean('init', MSs, size=(fwhm*1.5,fwhm*1.5), res='normal')
        ###

        ### group into patches corresponding to the mask islands
        mask_cl = mosaic_image.imagename.replace('image.fits', 'mask-cl.fits')
        # this mask is with no user region, done to isolate only bight compact sources
        if not os.path.exists(mask_cl):
            mosaic_image.beamReg = 'ddcal/beam.reg'
            mosaic_image.makeMask(threshpix=7,
                                  atrous_do=False,
                                  remove_extended_cutoff=0.001,
                                  maskname=mask_cl,
                                  only_beam=True)

        lsm = lsmtool.load(mosaic_image.skymodel_cut)
        lsm.group(mask_cl, root='Isl')
        # this removes all sources not in the mask-cl
        lsm.select('Patch = Isl.*', useRegEx=True)
        # this regroup sources
        x = lsm.getColValues('RA', aggregate='wmean')
        y = lsm.getColValues('Dec', aggregate='wmean')
        flux = lsm.getColValues('I', aggregate='sum')
        grouper = lib_dd_parallel.Grouper(list(zip(x, y)),
                                          flux,
                                          look_distance=0.3,
                                          kernel_size=0.1,
                                          grouping_distance=0.05)
        grouper.run()
        clusters = grouper.grouping()
        grouper.plot()
Esempio n. 26
0
phasecentre = MSs.getListObj()[0].getPhaseCentre()
MSs.getListObj()[0].makeBeamReg('self/beam.reg')
beamReg = 'self/beam.reg'

#################################################################
# Get online model
if sourcedb is None:
    if not os.path.exists('tgts.skydb'):
        fwhm = MSs.getListObj()[0].getFWHM()
        radeg = phasecentre[0]
        decdeg = phasecentre[1]
        # get model the size of the image (radius=fwhm/2)
        os.system(
            'wget -O tgts.skymodel "http://172.104.228.177/cgi-bin/gsmv1.cgi?coord=%f,%f&radius=%f"'
            % (radeg, decdeg, fwhm))
        lsm = lsmtool.load('tgts.skymodel')  #, beamMS=MSs.getListObj()[0])
        #Reduces the flux of clean component according to a primary beam function
        #NOTE: monochromatic approximation!
        center = SkyCoord(phasecentre[0] * u.deg, phasecentre[1] * u.deg)
        sources = SkyCoord(
            lsm.getColValues('RA') * u.deg,
            lsm.getColValues('Dec') * u.deg)
        d = center.separation(sources)
        # from http://www.aips.nrao.edu/cgi-bin/ZXHLP2.PL?PBCOR (converto to arcmin and multiply by freq in GHz)
        d = d.deg * 60 * np.mean(MSs.getListObj()[0].getFreqs()) / 1.e9
        I = lsm.getColValues('I')
        parm = [-3.397, 47.192, -30.931, 7.803]  # 325 MHz GMRT
        I_corr = I * (1 + (parm[0]/10**3)*d**2 + (parm[1]/10**7)*d**4 + \
             (parm[2]/10**10)*d**6 + (parm[3]/10**13)*d**8)
        lsm.setColValues('I', I_corr)
        lsm.write('tgts.skymodel', clobber=True)
Esempio n. 27
0
def main(ms_input, SkymodelPath, Radius="5.", DoDownload="True", Source="TGSS"):
    """
    Download the skymodel for the target field

    Parameters
    ----------
    ms_input : str
        String from the list (map) of the target MSs
    SkymodelPath : str
        Full name (with path) to the skymodel; if YES is true, the skymodel will be downloaded here
    Radius : string with float (default = "5.")
        Radius for the TGSS/GSM cone search in degrees
    DoDownload : str ("Force" or "True" or "False")
        Download or not the TGSS skymodel or GSM.
        "Force": download skymodel from TGSS or GSM, delete existing skymodel if needed.
        "True" or "Yes": use existing skymodel file if it exists, download skymodel from
                         TGSS or GSM if it does not.
        "False" or "No": Do not download skymodel, raise an exception if skymodel
                         file does not exist.

    """

    FileExists = os.path.isfile(SkymodelPath)
    if (not FileExists and os.path.exists(SkymodelPath)):
        raise ValueError("download_tgss_skymodel_target: Path: \"%s\" exists but is not a file!"%(SkymodelPath))
    download_flag = False
    if not os.path.exists(os.path.dirname(SkymodelPath)):
        os.makedirs(os.path.dirname(SkymodelPath))
    if DoDownload.upper() == "FORCE":
        if FileExists:
            os.remove(SkymodelPath)
        download_flag = True
    elif DoDownload.upper() == "TRUE" or DoDownload.upper() == "YES":
        if FileExists:
            print("USING the exising skymodel in "+ SkymodelPath)
            return(0)
        else:
            download_flag = True
    elif DoDownload.upper() == "FALSE" or DoDownload.upper() == "NO":
         if FileExists:
            print("USING the exising skymodel in "+ SkymodelPath)
            return(0)
         else:
            raise ValueError("download_tgss_skymodel_target: Path: \"%s\" does not exist and skymodel download is disabled!"%(SkymodelPath))

    # If we got here, then we are supposed to download the skymodel.
    assert download_flag is True # Jaja, belts and suspenders...
    print("DOWNLOADING skymodel for the target into "+ SkymodelPath)

    # Reading a MS to find the coordinate (pyrap)
    [RATar,DECTar]=grab_coord_MS(input2strlist_nomapfile(ms_input)[0])

    # Downloading the skymodel, skip after five tries
    errorcode = 1
    tries     = 0
    while errorcode != 0 and tries < 5:
        if Source == 'TGSS':
            errorcode = os.system("wget -O "+SkymodelPath+ " \'http://tgssadr.strw.leidenuniv.nl/cgi-bin/gsmv4.cgi?coord="+str(RATar)+","+str(DECTar)+"&radius="+str(Radius)+"&unit=deg&deconv=y\' ")
        elif Source == 'GSM':
            errorcode = os.system("wget -O "+SkymodelPath+ " \'https://lcs165.lofar.eu/cgi-bin/gsmv1.cgi?coord="+str(RATar)+","+str(DECTar)+"&radius="+str(Radius)+"&unit=deg&deconv=y\' ")
        time.sleep(5)
        tries += 1

    if not os.path.isfile(SkymodelPath):
        raise IOError("download_tgss_skymodel_target: Path: \"%s\" does not exist after trying to download the skymodel."%(SkymodelPath))

    # Treat all sources as one group (direction)
    skymodel = lsmtool.load(SkymodelPath)
    skymodel.group('single')
    skymodel.write(clobber=True)
    
    return(0)
Esempio n. 28
0
    def selfCalRunFuncStatistics(self):	
		
		
			############################################################
			# Extract Statistics from D.Rafferty LSMtool Module
			############################################################
		
			# Create stats directory without flux cut 
			plotsGSMDir_noFluxCut				= self.statDir+'LSMTools/GSM_Comparison_noFluxCut/'
			plotNVSSDir_noFluxCut				= self.statDir+'LSMTools/NVSS_Comparison_noFluxCut/'

			cmd1 = 'mkdir -p %s'%(plotsGSMDir_noFluxCut)
			cmd2 = 'mkdir -p %s'%(plotNVSSDir_noFluxCut)
			os.system(cmd1)
			os.system(cmd2)
			
			if self.i != 0:
					plotPreviousIterDir_noFluxCut		= self.statDir+'LSMTools/Iter%s_Iter%s_Comparison_noFluxCut/'%(self.i-1,self.i)
					cmd3 ='mkdir -p %s'%(plotPreviousIterDir_noFluxCut)
					os.system(cmd3)
					
					
			# Create stats directory with flux cut I>500 mJy
			plotsGSMDir_500mJyCut				= self.statDir+'LSMTools/GSM_Comparison_500mJyCut/'
			plotNVSSDir_500mJyCut				= self.statDir+'LSMTools/NVSS_Comparison_500mJyCut/'

			cmd1 = 'mkdir -p %s'%(plotsGSMDir_500mJyCut)
			cmd2 = 'mkdir -p %s'%(plotNVSSDir_500mJyCut)
			os.system(cmd1)
			os.system(cmd2)
			
			if self.i != 0:
					plotPreviousIterDir_500mJyCut		= self.statDir+'LSMTools/Iter%s_Iter%s_Comparison_500mJyCut/'%(self.i-1,self.i)
					cmd3 ='mkdir -p %s'%(plotPreviousIterDir_500mJyCut)
					os.system(cmd3)

		
					
			# Load Skymodels					
			pybdsm_model_current 	= lsmtool.load(self.statisticsSkymodelCurrent)
			if self.i != 0:		
				pybdsm_model_previous	= lsmtool.load(self.statisticsSkymodelPrevious)
							
			gsm_model 				= lsmtool.load('gsm', VOPosition=[self.ra_target, self.dec_target], VORadius='%s degree'%(self.FOV))
			nvss_model 				= lsmtool.load('nvss', VOPosition=[self.ra_target, self.dec_target], VORadius='%s degree'%(self.FOV))		
					
					
					
			# Run comparison no flux cut		
			pybdsm_model_current.compare(gsm_model, outDir='%s'%(plotsGSMDir_noFluxCut), radius='2 arcmin')		
			pybdsm_model_current.compare(nvss_model, outDir='%s'%(plotNVSSDir_noFluxCut), radius='2 arcmin')
			if self.i != 0:
					pybdsm_model_current.compare(pybdsm_model_previous, outDir='%s'%(plotPreviousIterDir_noFluxCut), radius='2 arcmin')		


			# Run comparison with flux cut I>500mJy
			pybdsm_model_current.select('I > 500 mJy')
			if self.i != 0:
				pybdsm_model_previous.select('I > 500 mJy')

			pybdsm_model_current.compare(gsm_model, outDir='%s'%(plotsGSMDir_500mJyCut), radius='2 arcmin', excludeByFlux=True)		
			pybdsm_model_current.compare(nvss_model, outDir='%s'%(plotNVSSDir_500mJyCut), radius='2 arcmin', excludeByFlux=True)
			if self.i != 0:
					pybdsm_model_current.compare(pybdsm_model_previous, outDir='%s'%(plotPreviousIterDir_500mJyCut), radius='2 arcmin', excludeByFlux=True)	

		
			############################################################
			# Extract Instrument parmdb plots from Sarrvesh Module
			############################################################


			# generate instruments table plots 
			
			list_instrument_MS 	= range(len(self.Files))
			list_MS 			= range(len(self.Files))
			
			for k in range(len(self.Files)):
				if self.outerfovclean =='no':
					list_instrument_MS[k] 	= """%s%s/instrument"""%(self.IterDir,self.Files[k])
					list_MS[k]				= """%s"""%(self.Files[k])	
				if self.outerfovclean =='yes':
					list_instrument_MS[k] 	= """%s%s_sub%s/instrument"""%(self.IterDir,self.Files[k],self.preprocessIndex)
					list_MS[k]				= """%s_sub%s"""%(self.Files[k],self.preprocessIndex)
				
				
				parmdbOutputDir = """%s/Parmdb_Plots/%s/"""%(self.statDir,list_MS[k])		
				cmd 			= """mkdir -p %s"""%(parmdbOutputDir)				
				os.system(cmd)
				
				self.makePhasePlots("""%s"""%(list_instrument_MS[k]),"""%s"""%(parmdbOutputDir))
				
				

			############################################################
			# Other Stats extracted with pybdsm
			############################################################
			
			pybdsmStatDir		= """%sPybdsm_Stats/"""%(self.statDir)
			cmd_pybdsmStats 	= """mkdir -p %s"""%(pybdsmStatDir)
			
			os.system(cmd_pybdsmStats) 
			
			pybdsmStatFile		= """%sExtractedStats"""%(pybdsmStatDir)
			
			filepybdsm			= open(pybdsmStatFile,'w')
			
			cmd_pybdsm	= """
******************************************************************************
Informations extracted with pybdsm

- The (3-sigma) clipped rms = %s Jy 

- The Mean flux on the image = %s Jy

- The Total Flux on the image = %s Jy

						"""%(self.rmsclipped,self.Mean,self.TotalFlux)
			filepybdsm.write(cmd_pybdsm)
			filepybdsm.close()						
Esempio n. 29
0
def main(h5parmfile,
         soltabname='phase000',
         outroot='',
         bounds_deg=None,
         bounds_mid_deg=None,
         skymodel=None,
         solsetname='sol000',
         ressoltabname='',
         padding_fraction=1.4,
         cellsize_deg=0.1,
         smooth_deg=0,
         gsize_deg=0,
         time_avg_factor=1,
         fasth5parm=None,
         interp_kind='nearest'):
    """
    Make a-term FITS images

    Parameters
    ----------
    h5parmfile : str
        Filename of h5parm
    soltabname : str
        Soltab containing solutions or screen fit
    outroot : str
        Root of filename of output FITS file (root+'_0.fits')
    bounds_deg : list
        List of [maxRA, minDec, minRA, maxDec] for image bounds
    bounds_mid_deg : list
        List of [RA, Dec] for midpoint of image bounds
    skymodel : str
        Filename of calibration sky model (needed for patch positions)
    solsetname : str, optional
        Name of solset
    ressoltabname : str, optional
        Soltab containing the screen residuals
    padding_fraction : float, optional
        Fraction of total size to pad with (e.g., 0.2 => 20% padding all around)
    cellsize_deg : float, optional
        Cellsize of output image
    smooth_deg : float, optional
        Size of smoothing kernel in degrees to apply
    gsize_deg : float, optional
        FWHM in degrees of Gaussian to add at patch locations (to enforce that
        solutions at these locations are exactly equal to those in the h5parm)
    time_avg_factor : int, optional
        Averaging factor in time for fast-phase corrections
    fasth5parm : str, optional
        Filename of fast-phase h5parm to be added together with input h5parm
        (interpolation of the input h5parm is done first)
    interp_kind : str, optional
        Kind of interpolation to use, if fasth5parm is given. Can be any
        supported by scipy.interpolate.interp1d

    Returns
    -------
    result : dict
        Dict with list of FITS files
    """
    # Read in solutions
    H = h5parm(h5parmfile)
    solset = H.getSolset(solsetname)
    if 'gain' in soltabname:
        soltab = solset.getSoltab(soltabname.replace('gain', 'amplitude'))
        soltab_ph = solset.getSoltab(soltabname.replace('gain', 'phase'))
    else:
        soltab = solset.getSoltab(soltabname)

    if type(bounds_deg) is str:
        bounds_deg = [
            float(f.strip()) for f in bounds_deg.strip('[]').split(';')
        ]
    if type(bounds_mid_deg) is str:
        bounds_mid_deg = [
            float(f.strip()) for f in bounds_mid_deg.strip('[]').split(';')
        ]
    if padding_fraction is not None:
        padding_fraction = float(padding_fraction)
        padding_ra = (bounds_deg[2] - bounds_deg[0]) * (padding_fraction - 1.0)
        padding_dec = (bounds_deg[3] - bounds_deg[1]) * (padding_fraction -
                                                         1.0)
        bounds_deg[0] -= padding_ra
        bounds_deg[1] -= padding_dec
        bounds_deg[2] += padding_ra
        bounds_deg[3] += padding_dec
    cellsize_deg = float(cellsize_deg)
    gsize_deg = float(gsize_deg)
    gsize_pix = gsize_deg / cellsize_deg
    smooth_deg = float(smooth_deg)
    smooth_pix = smooth_deg / cellsize_deg
    time_avg_factor = int(time_avg_factor)

    # Do Voronoi tessellation + smoothing
    if 'amplitude' in soltab.getType():
        # complexgain
        vals = soltab.val
        vals_ph = soltab_ph.val
    else:
        # scalarphase -> set amplitudes to unity
        vals_ph = soltab.val
        vals = np.ones_like(vals_ph)
    times = soltab.time
    freqs = soltab.freq
    ants = soltab.ant
    axis_names = soltab.getAxesNames()
    source_names = soltab.dir[:]

    # Load fast-phase solutions if needed and combine with slow gains by interpolating
    # the slow gains to the fast time grid
    if 'amplitude' in soltab.getType() and fasth5parm is not None:
        H_fast = h5parm(fasth5parm)
        solset_fast = H_fast.getSolset('sol000')
        soltab_fast = solset_fast.getSoltab('phase000')
        times_fast = soltab_fast.time
        freqs_fast = soltab_fast.freq

        # Interpolate the slow gains to the fast times and frequencies
        axis_names = soltab.getAxesNames()
        time_ind = axis_names.index('time')
        freq_ind = axis_names.index('freq')
        if len(times) == 1:
            # If just a single time, we just repeat the values as needed
            fast_axis_names = soltab_fast.getAxesNames()
            fast_time_ind = fast_axis_names.index('time')
            fast_freq_ind = fast_axis_names.index('freq')
            new_shape = list(vals.shape)
            new_shape[time_ind] = soltab_fast.val.shape[fast_time_ind]
            new_shape[freq_ind] = soltab_fast.val.shape[fast_freq_ind]
            vals = np.resize(vals, new_shape)
            vals_ph = np.resize(vals, new_shape)
        else:
            # Interpolate (in log space for amps)
            logvals = np.log10(vals)
            f = si.interp1d(times,
                            logvals,
                            axis=time_ind,
                            kind=interp_kind,
                            fill_value='extrapolate')
            logvals = f(times_fast)
            f = si.interp1d(freqs,
                            logvals,
                            axis=freq_ind,
                            kind=interp_kind,
                            fill_value='extrapolate')
            logvals = f(freqs_fast)
            vals = 10**(logvals)
            f = si.interp1d(times,
                            vals_ph,
                            axis=time_ind,
                            kind=interp_kind,
                            fill_value='extrapolate')
            vals_ph = f(times_fast)
            f = si.interp1d(freqs,
                            vals_ph,
                            axis=freq_ind,
                            kind=interp_kind,
                            fill_value='extrapolate')
            vals_ph = f(freqs_fast)
        for p in range(2):
            vals_ph[:, :, :, :, p] += soltab_fast.val
        freqs = freqs_fast
        times = times_fast

    # Make blank output FITS file (type does not matter at this point)
    midRA = bounds_mid_deg[0]
    midDec = bounds_mid_deg[1]
    temp_image = outroot + '.tmp'
    imsize = (bounds_deg[3] - bounds_deg[1])  # deg
    imsize = int(imsize / cellsize_deg)  # pix
    misc.make_template_image(temp_image,
                             midRA,
                             midDec,
                             ximsize=imsize,
                             yimsize=imsize,
                             cellsize_deg=cellsize_deg,
                             freqs=freqs,
                             times=[0.0],
                             antennas=soltab.ant,
                             aterm_type='tec')
    hdu = pyfits.open(temp_image, memmap=False)
    data = hdu[0].data
    w = wcs.WCS(hdu[0].header)
    RAind = w.axis_type_names.index('RA')
    Decind = w.axis_type_names.index('DEC')

    # Get x, y coords for directions in pixels. We use the input calibration sky
    # model for this, as the patch positions written to the h5parm file by DPPP may
    # be different
    skymod = lsmtool.load(skymodel)
    source_dict = skymod.getPatchPositions()
    source_positions = []
    for source in source_names:
        radecpos = source_dict[source.strip('[]')]
        source_positions.append([radecpos[0].value, radecpos[1].value])
    source_positions = np.array(source_positions)
    ra_deg = source_positions.T[0]
    dec_deg = source_positions.T[1]
    xy = []
    for RAvert, Decvert in zip(ra_deg, dec_deg):
        ra_dec = np.array([[0.0, 0.0, 0.0, 0.0, 0.0]])
        ra_dec[0][RAind] = RAvert
        ra_dec[0][Decind] = Decvert
        xy.append((w.wcs_world2pix(ra_dec, 0)[0][RAind],
                   w.wcs_world2pix(ra_dec, 0)[0][Decind]))

    # Get boundary for imaging region in pixels
    ra_dec = np.array([[0.0, 0.0, 0.0, 0.0, 0.0]])
    ra_dec[0][RAind] = bounds_deg[0]
    ra_dec[0][Decind] = bounds_deg[1]
    field_minxy = (w.wcs_world2pix(ra_dec,
                                   0)[0][RAind], w.wcs_world2pix(ra_dec,
                                                                 0)[0][Decind])
    ra_dec[0][RAind] = bounds_deg[2]
    ra_dec[0][Decind] = bounds_deg[3]
    field_maxxy = (w.wcs_world2pix(ra_dec,
                                   0)[0][RAind], w.wcs_world2pix(ra_dec,
                                                                 0)[0][Decind])

    # Generate array of outer points used to constrain the facets
    nouter = 64
    means = np.ones((nouter, 2)) * np.array(xy).mean(axis=0)
    offsets = []
    angles = [np.pi / (nouter / 2.0) * i for i in range(0, nouter)]
    for ang in angles:
        offsets.append([np.cos(ang), np.sin(ang)])
    radius = 2.0 * np.sqrt((field_maxxy[0] - field_minxy[0])**2 +
                           (field_maxxy[1] - field_minxy[1])**2)
    scale_offsets = radius * np.array(offsets)
    outer_box = means + scale_offsets

    # Tessellate and clip
    points_all = np.vstack([xy, outer_box])
    vor = Voronoi(points_all)
    lines = [
        shapely.geometry.LineString(vor.vertices[line])
        for line in vor.ridge_vertices if -1 not in line
    ]
    polygons = [poly for poly in shapely.ops.polygonize(lines)]

    # Index polygons to directions
    ind = []
    for i, xypos in enumerate(xy):
        for poly in polygons:
            if poly.contains(Point(xypos)):
                poly.index = i

    # Rasterize the polygons to an array, with the value being equal to the
    # polygon's index+1
    data_template = np.ones(data[0, 0, 0, :, :].shape)
    data_rasertize_template = np.zeros(data[0, 0, 0, :, :].shape)
    for poly in polygons:
        verts_xy = poly.exterior.xy
        verts = []
        for x, y in zip(verts_xy[0], verts_xy[1]):
            verts.append((x, y))
        poly_raster = misc.rasterize(verts,
                                     data_template.copy()) * (poly.index + 1)
        filled = np.where(poly_raster > 0)
        data_rasertize_template[filled] = poly_raster[filled]

    # Identify any duplicate times and remove
    delta_times = times[1:] - times[:-1]  # time at center of solution interval
    nodupind = np.where(delta_times > 0.1)
    times = times[nodupind]
    if 'pol' in axis_names:
        vals = np.squeeze(vals[nodupind, :, :, :, :])
        vals_ph = np.squeeze(vals_ph[nodupind, :, :, :, :])
    else:
        vals = np.squeeze(vals[nodupind, :, :, :])
        vals_ph = np.squeeze(vals_ph[nodupind, :, :, :])

    # Identify any gaps in time (frequency gaps are not allowed), as we need to
    # output a separate FITS file for each time chunk
    delta_times = times[1:] - times[:-1]  # time at center of solution interval
    timewidth = np.min(delta_times)
    gaps = np.where(delta_times > timewidth * 1.2)
    gaps_ind = gaps[0] + 1
    gaps_ind = np.append(gaps_ind, np.array([len(times)]))

    # Add additional breaks to gaps_ind to keep memory use within that available
    # From experience, making a (30, 46, 62, 4, 146, 146) aterm image needs around
    # 30 GB of memory
    if soltab.getType() == 'tec':
        max_ntimes = 15 * 46 * 4
    else:
        max_ntimes = 15
    # TODO: adjust max_ntimes depending on available memory and time_avg_factor
    check_gaps = True
    while check_gaps:
        check_gaps = False
        g_start = 0
        gaps_ind_copy = gaps_ind.copy()
        for gnum, g_stop in enumerate(gaps_ind_copy):
            if g_stop - g_start > max_ntimes:
                new_gap = g_start + int((g_stop - g_start) / 2)
                gaps_ind = np.insert(gaps_ind, gnum, np.array([new_gap]))
                check_gaps = True
                break
            g_start = g_stop

    if soltab.getType() == 'tec':
        # TEC solutions
        # input data are [time, ant, dir, freq]
        # output data are [RA, DEC, ANTENNA, FREQ, TIME].T
        # Now loop over stations, frequencies, and times and fill in the correct
        # values
        outfiles = []
        g_start = 0
        for gnum, g_stop in enumerate(gaps_ind):
            outfile = '{0}_{1}.fits'.format(outroot, gnum)
            misc.make_template_image(temp_image,
                                     midRA,
                                     midDec,
                                     ximsize=imsize,
                                     yimsize=imsize,
                                     cellsize_deg=cellsize_deg,
                                     times=times[g_start:g_stop],
                                     freqs=freqs,
                                     antennas=soltab.ant,
                                     aterm_type='tec')
            hdu = pyfits.open(temp_image, memmap=False)
            data = hdu[0].data
            w = wcs.WCS(hdu[0].header)
            for t, time in enumerate(times[g_start:g_stop]):
                for f, freq in enumerate(freqs):
                    for s, stat in enumerate(ants):
                        for poly in polygons:
                            ind = np.where(
                                data_rasertize_template == poly.index + 1)
                            data[t, f, s, ind[0],
                                 ind[1]] = vals[t + g_start, s, poly.index, 0]

                        # Smooth if desired
                        if smooth_pix > 0:
                            data[t, f, s, :, :] = ndimage.gaussian_filter(
                                data[t, f, s, :, :],
                                sigma=(smooth_pix, smooth_pix),
                                order=0)

                        # Add Gaussians at patch positions if desired
                        if gsize_pix > 0:
                            for i, (x, y) in enumerate(xy):
                                # Only do this if patch is inside the region of interest
                                if int(x) >= 0 and int(
                                        x) < data.shape[4] and int(
                                            y) >= 0 and int(y) < data.shape[3]:
                                    A = vals[t + g_start, s, i,
                                             0] - data[t, f, s,
                                                       int(y),
                                                       int(x)]
                                    data[t, f, s, :, :] += guassian_image(
                                        A, x, y, data.shape[4], data.shape[3],
                                        gsize_pix)
            g_start = g_stop

            # Write FITS file
            hdu[0].data = data
            hdu.writeto(outfile, overwrite=True)
            outfiles.append(outfile)
            os.remove(temp_image)

        outfile = open(outroot + '.txt', 'w')
        outfile.writelines([o + '\n' for o in outfiles])
        outfile.close()

    else:
        # Gain solutions
        # input data are [time, freq, ant, dir, pol] for slow gains (complexgain)
        # and [time, freq, ant, dir] for fast (non-tec) phases (scalarphase)
        # output data are [RA, DEC, MATRIX, ANTENNA, FREQ, TIME].T
        # Now loop over stations, frequencies, and times and fill in the correct
        # matrix values (matrix dimension has 4 elements: real XX, imaginary XX,
        # real YY and imaginary YY)
        outfiles = []
        g_start = 0
        for gnum, g_stop in enumerate(gaps_ind):
            outfile = '{0}_{1:04}.fits'.format(outroot, gnum)
            misc.make_template_image(temp_image,
                                     midRA,
                                     midDec,
                                     ximsize=imsize,
                                     yimsize=imsize,
                                     cellsize_deg=cellsize_deg,
                                     times=times[g_start:g_stop],
                                     freqs=freqs,
                                     antennas=soltab.ant,
                                     aterm_type='gain')
            hdu = pyfits.open(temp_image, memmap=False)
            data = hdu[0].data
            w = wcs.WCS(hdu[0].header)
            for t, time in enumerate(times[g_start:g_stop]):
                for f, freq in enumerate(freqs):
                    for s, stat in enumerate(ants):
                        for p, poly in enumerate(polygons):
                            ind = np.where(
                                data_rasertize_template == poly.index + 1)
                            if 'pol' in axis_names:
                                val_amp_xx = vals[t + g_start, f, s,
                                                  poly.index, 0]
                                val_amp_yy = vals[t + g_start, f, s,
                                                  poly.index, 1]
                                val_phase_xx = vals_ph[t + g_start, f, s,
                                                       poly.index, 0]
                                val_phase_yy = vals_ph[t + g_start, f, s,
                                                       poly.index, 1]
                            else:
                                val_amp_xx = vals[t + g_start, f, s,
                                                  poly.index]
                                val_amp_yy = vals[t + g_start, f, s,
                                                  poly.index]
                                val_phase_xx = vals_ph[t + g_start, f, s,
                                                       poly.index]
                                val_phase_yy = vals_ph[t + g_start, f, s,
                                                       poly.index]
                            data[t, f, s, 0, ind[0],
                                 ind[1]] = val_amp_xx * np.cos(val_phase_xx)
                            data[t, f, s, 2, ind[0],
                                 ind[1]] = val_amp_yy * np.cos(val_phase_yy)
                            data[t, f, s, 1, ind[0],
                                 ind[1]] = val_amp_xx * np.sin(val_phase_xx)
                            data[t, f, s, 3, ind[0],
                                 ind[1]] = val_amp_yy * np.sin(val_phase_yy)

                        # Smooth if desired
                        if smooth_pix > 0:
                            data[t, f, s, :, :, :] = ndimage.gaussian_filter(
                                data[t, f, s, :, :, :],
                                sigma=(0, smooth_pix, smooth_pix),
                                order=0)

                        # Add Gaussians at patch positions if desired
                        if gsize_pix > 0:
                            for i, (x, y) in enumerate(xy):
                                # Only do this if patch is inside the region of interest
                                if int(x) >= 0 and int(
                                        x) < data.shape[4] and int(
                                            y) >= 0 and int(y) < data.shape[3]:
                                    if 'pol' in axis_names:
                                        val_amp_xx = vals[t + g_start, f, s, i,
                                                          0]
                                        val_amp_yy = vals[t + g_start, f, s, i,
                                                          1]
                                        val_phase_xx = vals_ph[t + g_start, f,
                                                               s, i, 0]
                                        val_phase_yy = vals_ph[t + g_start, f,
                                                               s, i, 1]
                                    else:
                                        val_amp_xx = vals[t + g_start, f, s, i]
                                        val_amp_yy = vals[t + g_start, f, s, i]
                                        val_phase_xx = vals_ph[t + g_start, f,
                                                               s, i]
                                        val_phase_yy = vals_ph[t + g_start, f,
                                                               s, i]
                                    A = val_amp_xx * np.cos(
                                        val_phase_xx) - data[t, f, s, 0,
                                                             int(y),
                                                             int(x)]
                                    data[t, f, s, 0, :, :] += guassian_image(
                                        A, x, y, data.shape[5], data.shape[4],
                                        gsize_pix)
                                    A = val_amp_yy * np.cos(
                                        val_phase_yy) - data[t, f, s, 2,
                                                             int(y),
                                                             int(x)]
                                    data[t, f, s, 2, :, :] += guassian_image(
                                        A, x, y, data.shape[5], data.shape[4],
                                        gsize_pix)
                                    A = val_amp_xx * np.sin(
                                        val_phase_xx) - data[t, f, s, 1,
                                                             int(y),
                                                             int(x)]
                                    data[t, f, s, 1, :, :] += guassian_image(
                                        A, x, y, data.shape[5], data.shape[4],
                                        gsize_pix)
                                    A = val_amp_yy * np.sin(
                                        val_phase_yy) - data[t, f, s, 3,
                                                             int(y),
                                                             int(x)]
                                    data[t, f, s, 3, :, :] += guassian_image(
                                        A, x, y, data.shape[5], data.shape[4],
                                        gsize_pix)

            # If averaging in time, make a new template image with
            # fewer times and write to that instead
            if time_avg_factor > 1:
                times_avg = times[g_start:g_stop:time_avg_factor]
                ntimes = len(times_avg)
                misc.make_template_image(temp_image + '.avg',
                                         midRA,
                                         midDec,
                                         ximsize=imsize,
                                         yimsize=imsize,
                                         cellsize_deg=cellsize_deg,
                                         times=times_avg,
                                         freqs=freqs,
                                         antennas=soltab.ant,
                                         aterm_type='gain')
                hdu = pyfits.open(temp_image + '.avg', memmap=False)
                data_avg = hdu[0].data

                # Average
                for t, time in enumerate(times_avg):
                    incr = min(
                        time_avg_factor,
                        len(times[g_start:g_stop]) - t * time_avg_factor)
                    data_avg[t, :, :, :, :, :] = np.nanmean(
                        data[t:t + incr, :, :, :, :, :], axis=0)
                data = data_avg
            else:
                ntimes = len(times[g_start:g_stop])

            # Ensure there are no NaNs in the images, as WSClean will produced uncorrected,
            # uncleaned images if so. We replace NaNs with 1.0 and 0.0 for real and
            # imaginary parts, respectively
            # Note: we iterate over time to reduce memory usage
            for t in range(ntimes):
                for p in range(4):
                    if p % 2:
                        # Imaginary elements
                        nanval = 0.0
                    else:
                        # Real elements
                        nanval = 1.0
                    data[t, :, :, p, :, :][np.isnan(data[t, :, :,
                                                         p, :, :])] = nanval

            # Write FITS file
            hdu[0].data = data
            hdu.writeto(outfile, overwrite=True)
            outfiles.append(outfile)
            os.remove(temp_image)
            hdu = None
            data = None

            # Update start time index
            g_start = g_stop

        outfile = open(outroot + '.txt', 'w')
        outfile.writelines([o + '\n' for o in outfiles])
        outfile.close()
Esempio n. 30
0
# Runs steps for validation

import lsmtool


s = lsmtool.load('tests/no_patches.sky')

print('Select individual sources with Stokes I fluxes above 1 Jy')
s.select('I > 1.0 Jy')

print('Transfer patches from patches.sky')
s.transfer('tests/patches.sky')

print('Remove patches with total fluxes below 2 Jy')
s.remove('I < 2.0 Jy', aggregate='sum')

print('Ungroup the skymodel')
s.ungroup()

print('Concatenate with concat.sky')
s.concatenate('tests/concat.sky', matchBy = 'position', radius = '30 arcsec', keep = 'from2')

print('Compare to concat.sky')
c = lsmtool.load('tests/concat.sky')
c.ungroup()
c.select('I > 5.0 Jy')
s.ungroup()
s.compare(c, outDir='tests/')

print('Add a source')
s.add({'Name': 'src1', 'Type': 'POINT', 'Ra': 277.4232, 'Dec': 48.3689, 'I': 0.69})
Esempio n. 31
0
            model_skymodel = 'ddcal/c%02i/skymodels/%s-best-source.txt' % (
                cmaj, d.name)
            model_skydb = 'ddcal/c%02i/skymodels/%s-best-source.skydb' % (
                cmaj, d.name)
            os.system('cp %s %s' %
                      (d.get_model('best') + '-sources.txt', model_skymodel))

            # restrict to initial mask
            logger.info('Restrict model to initial region')
            os.system('cp ' + d.model['best'] + '-mask.fits' + ' ' +
                      d.model['best'] + '-mask-restricted.fits')
            lib_img.blank_image_reg(d.model['best'] + '-mask-restricted.fits',
                                    d.get_region(),
                                    inverse=True,
                                    blankval=0.)
            lsm = lsmtool.load(model_skymodel)
            lsm.select('%s == True' %
                       (d.model['best'] + '-mask-restricted.fits'))
            lsm.write(model_skymodel, format='makesourcedb', clobber=True)

            lib_util.check_rm(model_skydb)
            s.add('makesourcedb outtype="blob" format="<" in="%s" out="%s"' %
                  (model_skymodel, model_skydb),
                  log='makesourcedb_cl.log',
                  commandType='general')
            s.run()

        # remove the DD-cal from original dataset using new solutions
        with w.if_todo('%s-subtract' % logstring):

            # Predict - ms:MODEL_DATA
Esempio n. 32
0
    def make_skymodel(self, index):
        """
        Makes predict sky model

        Parameters
        ----------
        index : int
            Iteration index
        """
        # First check whether sky model already exists due to a previous run and attempt
        # to load it if so
        dst_dir = os.path.join(self.field.working_dir, 'skymodels',
                               'predict_{}'.format(index))
        misc.create_directory(dst_dir)
        self.predict_skymodel_file = os.path.join(
            dst_dir, '{}_predict_skymodel.txt'.format(self.name))
        if os.path.exists(self.predict_skymodel_file):
            skymodel = lsmtool.load(str(self.predict_skymodel_file))
        else:
            # If sky model does not already exist, make it
            if self.is_outlier or self.is_bright_source:
                # For outlier and bright-source sectors, we use the sky model made earlier,
                # with no filtering
                skymodel = self.predict_skymodel
            else:
                # For imaging sectors, we use the full calibration sky model and filter it
                # to keep only sources inside the sector
                skymodel = self.calibration_skymodel.copy()
                skymodel = self.filter_skymodel(skymodel)

            # Remove the bright sources from the sky model if they will be predicted and
            # subtracted separately (so that they aren't subtracted twice)
            if self.field.peel_bright_sources and not self.is_outlier and not self.is_bright_source:
                source_names = skymodel.getColValues('Name')
                bright_source_names = self.field.bright_source_skymodel.getColValues(
                    'Name')
                matching_ind = []
                for i, sn in enumerate(source_names):
                    if sn in bright_source_names:
                        matching_ind.append(i)
                if len(matching_ind) > 0:
                    skymodel.remove(np.array(matching_ind))

            # Write filtered sky model to file for later prediction
            if len(skymodel) > 0:
                skymodel.write(self.predict_skymodel_file, clobber=True)
            else:
                # No sources, so just make a dummy sky model with single,
                # very faint source at center
                dummylines = [
                    "Format = Name, Type, Patch, Ra, Dec, I, SpectralIndex, LogarithmicSI, "
                    "ReferenceFrequency='100000000.0', MajorAxis, MinorAxis, Orientation\n"
                ]
                ra = misc.ra2hhmmss(self.ra)
                sra = str(ra[0]).zfill(2) + ':' + str(
                    ra[1]).zfill(2) + ':' + str("%.6f" % (ra[2])).zfill(6)
                dec = misc.dec2ddmmss(self.dec)
                decsign = ('-' if dec[3] < 0 else '+')
                sdec = decsign + str(dec[0]).zfill(2) + '.' + str(
                    dec[1]).zfill(2) + '.' + str("%.6f" % (dec[2])).zfill(6)
                patch = self.calibration_skymodel.getPatchNames()[0]
                dummylines.append(',,{0},{1},{2}\n'.format(patch, sra, sdec))
                dummylines.append('s0c0,POINT,{0},{1},{2},0.00000001,'
                                  '[0.0,0.0],false,100000000.0,,,\n'.format(
                                      patch, sra, sdec))
                with open(self.predict_skymodel_file, 'w') as f:
                    f.writelines(dummylines)
                skymodel = lsmtool.load(str(self.predict_skymodel_file))

        # Save list of patches (directions) in the format written by DDECal in the h5parm
        self.patches = ['[{}]'.format(p) for p in skymodel.getPatchNames()]

        # Find nearest patch to flux-weighted center of the sector sky model
        if not self.is_outlier and not self.is_bright_source:
            tmp_skymodel = skymodel.copy()
            tmp_skymodel.group('single')
            ra, dec = tmp_skymodel.getPatchPositions(method='wmean',
                                                     asArray=True)
            patch_dist = skymodel.getDistance(ra[0], dec[0],
                                              byPatch=True).tolist()
            patch_names = skymodel.getPatchNames()
            self.central_patch = patch_names[patch_dist.index(min(patch_dist))]

            # Filter the field source sky model and store source sizes
            all_source_names = self.field.source_skymodel.getColValues(
                'Name').tolist()
            source_names = skymodel.getColValues('Name')
            if len(source_names
                   ) == 1 and source_names[0] not in all_source_names:
                # This occurs when a dummy sky model was made above, so skip the size
                # determination below
                source_skymodel = []
            else:
                in_sector = np.array(
                    [all_source_names.index(sn) for sn in source_names])
                source_skymodel = self.field.source_skymodel.copy()
                source_skymodel.select(in_sector)
            if len(source_skymodel) > 0:
                self.source_sizes = source_skymodel.getPatchSizes(
                    units='degree')
            else:
                self.source_sizes = [0.0]

        # Set the parameters for predict
        self.set_prediction_parameters()
Esempio n. 33
0
# make beam to the first mid null
phasecentre = MSs.getListObj()[0].getPhaseCentre()
# MSs.getListObj()[0].makeBeamReg('bench/beam.reg', freq='mid', to_null=True)
# beamReg = 'bench/beam.reg'
#################################################################
# Get online model
if sourcedb == '':
    if not os.path.exists('bench/tgts_bench.skydb'):
        fwhm = MSs.getListObj()[0].getFWHM(freq='min')
        radeg = phasecentre[0]
        decdeg = phasecentre[1]
        # get model the size of the image (radius=fwhm/2)
        os.system(
            'wget -O bench/tgts_bench.skymodel "https://lcs165.lofar.eu/cgi-bin/gsmv1.cgi?coord=%f,%f&radius=%f&unit=deg"'
            % (radeg, decdeg, fwhm / 2.))  # ASTRON
        lsm = lsmtool.load(
            'bench/tgts_bench.skymodel')  #, beamMS=MSs.getListObj()[0])
        lsm.remove('I<1')
        lsm.write('bench/tgts_bench.skymodel', clobber=True)
        os.system(
            'makesourcedb outtype="blob" format="<" in=bench/tgts_bench.skymodel out=bench/tgts_bench.skydb'
        )
        apparent = False
    sourcedb = 'bench/tgts_bench.skydb'

#################################################################################################
# Add model to MODEL_DATA
# copy sourcedb into each MS to prevent concurrent access from multiprocessing to the sourcedb
sourcedb_basename = sourcedb.split('/')[-1]
for MS in MSs.getListStr():
    lib_util.check_rm(MS + '/' + sourcedb_basename)
    logger.debug('Copy: ' + sourcedb + ' -> ' + MS)
Esempio n. 34
0
    # make mask
    im = lib_img.Image(imagename + '-MFS-image.fits')
    im.makeMask(threshisl=3)

    logger.info('Cleaning w/ mask...')
    s.add('wsclean -continue -reorder -temp-dir /dev/shm -name ' + imagename + ' -size ' + str(size) + ' ' + str(size) + ' -j '+str(s.max_processors)+' -baseline-averaging 3 \
            -scale 5arcsec -weight briggs 0.0 -niter 100000 -no-update-model-required -minuv-l 30 -mgain 0.85 -clean-border 1 \
            -auto-threshold 0.1 -fits-mask '                                            +im.maskname+' \
            -join-channels -fit-spectral-pol 2 -channels-out 10 -save-source-list '                                                                                   +MSs.getStrWsclean(), \
            log='wscleanB.log', commandType='wsclean', processors = 'max')
    s.run(check=True)
    os.system('cat logs/wscleanB.log | grep "background noise"')

    # make new mask
    im.makeMask(threshisl=5)

    # apply mask
    import lsmtool
    logger.info('Predict (apply mask)...')
    lsm = lsmtool.load(imagename + '-sources.txt')
    lsm.select('%s == True' % (imagename + '-mask.fits'))
    cRA, cDEC = MSs.getListObj()[0].getPhaseCentre()
    lsm.select(lsm.getDistance(cRA, cDEC) > 0.1)  # remove very centra part
    lsm.group('every')
    lsm.write(imagename + '-sources-cut.txt',
              format='makesourcedb',
              clobber=True)
    del lsm

logger.info("Done.")
Esempio n. 35
0
def main(input_image, input_skymodel_pb, input_bright_skymodel_pb, output_root,
         vertices_file, threshisl=5.0, threshpix=7.5, rmsbox=(150, 50),
         rmsbox_bright=(35, 7), adaptive_rmsbox=True,
         use_adaptive_threshold=False, adaptive_thresh=75.0, beamMS=None,
         peel_bright=False):
    """
    Filter the input sky model so that they lie in islands in the image

    Parameters
    ----------
    input_image : str
        Filename of input image to use to detect sources for filtering. Ideally, this
        should be a flat-noise image (i.e., without primary-beam correction)
    input_skymodel_pb : str
        Filename of input makesourcedb sky model, with primary-beam correction
    input_bright_skymodel_pb : str
        Filename of input makesourcedb sky model of bright sources only, with primary-
        beam correction
    output_root : str
        Root of filename of output makesourcedb sky models. Output filenames will be
        output_root+'.apparent_sky.txt' and output_root+'.true_sky.txt'
    vertices_file : str
        Filename of file with vertices
    threshisl : float, optional
        Value of thresh_isl PyBDSF parameter
    threshpix : float, optional
        Value of thresh_pix PyBDSF parameter
    rmsbox : tuple of floats, optional
        Value of rms_box PyBDSF parameter
    rmsbox_bright : tuple of floats, optional
        Value of rms_box_bright PyBDSF parameter
    adaptive_rmsbox : tuple of floats, optional
        Value of adaptive_rms_box PyBDSF parameter
    use_adaptive_threshold : bool, optional
        If True, use an adaptive threshold estimated from the negative values in
        the image
    adaptive_thresh : float, optional
        If adaptive_rmsbox is True, this value sets the threshold above
        which a source will use the small rms box
    peel_bright : bool, optional
        If True, bright sources were peeled, so add then back before filtering
    """
    if rmsbox is not None and isinstance(rmsbox, str):
        rmsbox = eval(rmsbox)
    if isinstance(rmsbox_bright, str):
        rmsbox_bright = eval(rmsbox_bright)
    adaptive_rmsbox = misc.string2bool(adaptive_rmsbox)
    use_adaptive_threshold = misc.string2bool(use_adaptive_threshold)
    if isinstance(beamMS, str):
        beamMS = misc.string2list(beamMS)
    peel_bright = misc.string2bool(peel_bright)

    # Try to set the TMPDIR evn var to a short path, to ensure we do not hit the length
    # limits for socket paths (used by the mulitprocessing module). We try a number of
    # standard paths (the same ones used in the tempfile Python library)
    old_tmpdir = os.environ["TMPDIR"]
    for tmpdir in ['/tmp', '/var/tmp', '/usr/tmp']:
        if os.path.exists(tmpdir):
            os.environ["TMPDIR"] = tmpdir
            break

    # Run PyBDSF to make a mask for grouping
    if use_adaptive_threshold:
        # Get an estimate of the rms by running PyBDSF to make an rms map
        img = bdsf.process_image(input_image, mean_map='zero', rms_box=rmsbox,
                                 thresh_pix=threshpix, thresh_isl=threshisl,
                                 thresh='hard', adaptive_rms_box=adaptive_rmsbox,
                                 adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright,
                                 rms_map=True, quiet=True, stop_at='isl')

        # Find min and max pixels
        max_neg_val = abs(np.min(img.ch0_arr))
        max_neg_pos = np.where(img.ch0_arr == np.min(img.ch0_arr))
        max_pos_val = abs(np.max(img.ch0_arr))
        max_pos_pos = np.where(img.ch0_arr == np.max(img.ch0_arr))

        # Estimate new thresh_isl from min pixel value's sigma, but don't let
        # it get higher than 1/2 of the peak's sigma
        threshisl_neg = 2.0 * max_neg_val / img.rms_arr[max_neg_pos][0]
        max_sigma = max_pos_val / img.rms_arr[max_pos_pos][0]
        if threshisl_neg > max_sigma / 2.0:
            threshisl_neg = max_sigma / 2.0

        # Use the new threshold only if it is larger than the user-specified one
        if threshisl_neg > threshisl:
            threshisl = threshisl_neg

    img = bdsf.process_image(input_image, mean_map='zero', rms_box=rmsbox,
                             thresh_pix=threshpix, thresh_isl=threshisl,
                             thresh='hard', adaptive_rms_box=adaptive_rmsbox,
                             adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright,
                             atrous_do=True, atrous_jmax=3, rms_map=True, quiet=True)

    emptysky = False
    if img.nisl > 0:
        maskfile = input_image + '.mask'
        img.export_image(outfile=maskfile, clobber=True, img_type='island_mask')

        # Construct polygon needed to trim the mask to the sector
        header = pyfits.getheader(maskfile, 0)
        w = wcs.WCS(header)
        RAind = w.axis_type_names.index('RA')
        Decind = w.axis_type_names.index('DEC')
        vertices = misc.read_vertices(vertices_file)
        RAverts = vertices[0]
        Decverts = vertices[1]
        verts = []
        for RAvert, Decvert in zip(RAverts, Decverts):
            ra_dec = np.array([[0.0, 0.0, 0.0, 0.0]])
            ra_dec[0][RAind] = RAvert
            ra_dec[0][Decind] = Decvert
            verts.append((w.wcs_world2pix(ra_dec, 0)[0][RAind], w.wcs_world2pix(ra_dec, 0)[0][Decind]))

        hdu = pyfits.open(maskfile, memmap=False)
        data = hdu[0].data

        # Rasterize the poly
        data_rasertize = data[0, 0, :, :]
        data_rasertize = misc.rasterize(verts, data_rasertize)
        data[0, 0, :, :] = data_rasertize

        hdu[0].data = data
        hdu.writeto(maskfile, overwrite=True)

        # Now filter the sky model using the mask made above
        if len(beamMS) > 1:
            # Select the best MS for the beam attenuation
            ms_times = []
            for ms in beamMS:
                tab = pt.table(ms, ack=False)
                ms_times.append(np.mean(tab.getcol('TIME')))
                tab.close()
            ms_times_sorted = sorted(ms_times)
            mid_time = ms_times_sorted[int(len(ms_times)/2)]
            beam_ind = ms_times.index(mid_time)
        else:
            beam_ind = 0
        try:
            s = lsmtool.load(input_skymodel_pb, beamMS=beamMS[beam_ind])
        except astropy.io.ascii.InconsistentTableError:
            emptysky = True
        if peel_bright:
            try:
                # If bright sources were peeled before imaging, add them back
                s_bright = lsmtool.load(input_bright_skymodel_pb, beamMS=beamMS[beam_ind])

                # Rename the bright sources, removing the '_sector_*' added previously
                # (otherwise the '_sector_*' text will be added every iteration,
                # eventually making for very long source names)
                new_names = [name.split('_sector')[0] for name in s_bright.getColValues('Name')]
                s_bright.setColValues('Name', new_names)
                if not emptysky:
                    s.concatenate(s_bright)
                else:
                    s = s_bright
                    emptysky = False
            except astropy.io.ascii.InconsistentTableError:
                pass
        if not emptysky:
            s.select('{} == True'.format(maskfile))  # keep only those in PyBDSF masked regions
            if len(s) == 0:
                emptysky = True
            else:
                # Write out apparent and true-sky models
                del(img)  # helps reduce memory usage
                s.group(maskfile)  # group the sky model by mask islands
                s.write(output_root+'.true_sky.txt', clobber=True)
                s.write(output_root+'.apparent_sky.txt', clobber=True, applyBeam=True)
    else:
        emptysky = True

    if emptysky:
        # No sources cleaned/found in image, so just make a dummy sky model with single,
        # very faint source at center
        dummylines = ["Format = Name, Type, Patch, Ra, Dec, I, SpectralIndex, LogarithmicSI, "
                      "ReferenceFrequency='100000000.0', MajorAxis, MinorAxis, Orientation\n"]
        ra, dec = img.pix2sky((img.shape[-2]/2.0, img.shape[-1]/2.0))
        if ra < 0.0:
            ra += 360.0
        ra = misc.ra2hhmmss(ra)
        sra = str(ra[0]).zfill(2)+':'+str(ra[1]).zfill(2)+':'+str("%.6f" % (ra[2])).zfill(6)
        dec = misc.dec2ddmmss(dec)
        decsign = ('-' if dec[3] < 0 else '+')
        sdec = decsign+str(dec[0]).zfill(2)+'.'+str(dec[1]).zfill(2)+'.'+str("%.6f" % (dec[2])).zfill(6)
        dummylines.append(',,p1,{0},{1}\n'.format(sra, sdec))
        dummylines.append('s0c0,POINT,p1,{0},{1},0.00000001,'
                          '[0.0,0.0],false,100000000.0,,,\n'.format(sra, sdec))
        with open(output_root+'.apparent_sky.txt', 'w') as f:
            f.writelines(dummylines)
        with open(output_root+'.true_sky.txt', 'w') as f:
            f.writelines(dummylines)

    # Set the TMPDIR env var back to its original value
    os.environ["TMPDIR"] = old_tmpdir
Esempio n. 36
0
#! /usr/bin/env python
# Runs an example of each operation
import lsmtool
import os


s = lsmtool.load('tests/no_patches.sky')


def test_select():
    print('Select individual sources with Stokes I fluxes above 1 Jy')
    s.select('I > 1.0 Jy')
    assert len(s) == 965


def test_transfer():
    print('Transfer patches from patches.sky')
    s.transfer('tests/patches.sky')
    assert s.hasPatches


def test_remove():
    print('Remove patches with total fluxes below 2 Jy')
    s.remove('I < 2.0 Jy', aggregate='sum')
    assert len(s) == 433


def test_upgroup():
    print('Ungroup the skymodel')
    s.ungroup()
    assert ~s.hasPatches
Esempio n. 37
0
def skymodel_files_are_similar(file1, file2, atol, rtol, verbosity):
    """
    Compare two skymodel files. Skymodels are considered similar if their
    statistics are similar. This function uses LSMTool to determine the skymodel
    statistics. The skymodels are considered to be similar, if the relevant
    statistics differ less than ``atol`` or ``rtol``.
    :param str file1: Filename of the first skymodel
    :param str file2: Filename of the second skymodel
    :param float atol: Absolute tolerance threshold
    :param float rtol: Relative tolerance threshold
    :param int verbosity: Verbosity level, higher is more verbose
    :return bool: True is skymodels file are similar, else False
    """
    def compare_stats(stats, atol, rtol):
        """
        Compare statistics generated by LSMTool. Check if the flux ratio is
        close to 1, and that the position offsets are less than ``atol``
        :param float atol: Absolute tolerance used to compare position offsets
        :param float rtol: Relative tolerance used to compare flux ratio
        :return bool: True if statistics are similar, else False
        """
        if not np.isclose(stats["meanRatio"], 1, rtol=rtol):
            return False
        if not np.isclose(stats["meanRAOffsetDeg"], 0, atol=atol):
            return False
        if not np.isclose(stats["meanDecOffsetDeg"], 0, atol=atol):
            return False
        return True

    logger.debug("Comparing skymodels '%s' and '%s'", file1, file2)
    sm1 = lsmtool.load(file1)
    sm2 = lsmtool.load(file2)
    stats = sm1.compare(sm2)
    if not stats:
        agree = False
    else:
        # We're only interested in a few statistics
        stats = {
            key: stats[key]
            for key in ("meanRatio", "meanRAOffsetDeg", "meanDecOffsetDeg")
        }
        agree = compare_stats(stats, atol, rtol)
    if agree:
        logger.info("Skymodel files '%s' and '%s' are similar", file1, file2)
    else:
        if stats:
            if verbosity > 0:
                logger.error(
                    "Skymodel files '%s' and '%s' differ (atol=%s, rtol=%s): %s",
                    file1,
                    file2,
                    atol,
                    rtol,
                    stats,
                )
            else:
                logger.error(
                    "Skymodel files '%s' and '%s' differ (atol=%s, rtol=%s)",
                    file1,
                    file2,
                    atol,
                    rtol,
                )
        else:
            logger.error("Failed to compare skymodel files '%s' and '%s'",
                         file1, file2)
    return agree
Esempio n. 38
0
def _set_up_directions(parset, bands, dry_run=False, test_run=False,
    reset_directions=[], reset_operations=[]):
    """
    Sets up directions (facets)

    Parameters
    ----------
    parset : dict
        Parset containing processing parameters
    bands : list of Band instances
        Vis data
    dry_run : bool, optional
        If True, do not run pipelines. All parsets, etc. are made as normal
    test_run : bool, optional
        If True, use test settings. These settings are for testing purposes
        only and will not produce useful results
    reset_directions : list of str, optional
        List of direction names to be reset
    reset_operations : list of str, optional
        Llist of operations to be reset

    Returns
    -------
    directions : List of Direction instances
        All directions to be used by the run() function
    direction_groups : List of lists of Direction instances
        Groups of directions to be selfcal-ed

    """
    dir_parset = parset['direction_specific']

    max_radius_deg = dir_parset['max_radius_deg']
    ref_band = bands[-1]

    if dir_parset['faceting_skymodel'] is not None:
        import lsmtool
        log.info("Using {} as sky model for source avoidance and DDE calibrator "
            "selection (if desired)".format(dir_parset['faceting_skymodel']))
        initial_skymodel = lsmtool.load(dir_parset['faceting_skymodel'])
    else:
        log.info("Building local sky model for source avoidance and DDE calibrator "
            "selection (if desired)...")
        initial_skymodel = factor.directions.make_initial_skymodel(ref_band)

    log.info('Setting up directions...')
    directions = _initialize_directions(parset, initial_skymodel, ref_band,
        max_radius_deg=max_radius_deg, dry_run=dry_run)

    # Check with user
    if parset['interactive']:
        print("\nFacet and DDE calibrator regions saved. Please check that they\n"
            "are OK before continuing. You can edit the directions file and\n"
            "continue; FACTOR will pick up any changes to it. Note: if you\n"
            "choose not to continue and you let FACTOR generate the directions\n"
            "internally, you must delete the FACTOR-made directions file\n"
            "(dir_working/factor_directions.txt) before restarting if you want\n"
            "to FACTOR to regenerate it\n")
        prompt = "Continue processing (y/n)? "
        answ = raw_input(prompt)
        while answ.lower() not in  ['y', 'n', 'yes', 'no']:
            answ = raw_input(prompt)
        if answ.lower() in ['n', 'no']:
            log.info('Exiting...')
            sys.exit(0)
        else:
            # Continue processing, but first re-initialize the directions to
            # pick up any changes the user made to the directions file
            directions = _initialize_directions(parset, initial_skymodel,
                ref_band, max_radius_deg=max_radius_deg, dry_run=dry_run)

    # Warn user if they've specified a direction to reset that does not exist
    direction_names = [d.name for d in directions]
    for name in reset_directions:
        if name not in direction_names and name != 'field':
            log.warn('Direction {} was specified for resetting but does not '
                'exist in current list of directions'.format(name))

    # Load previously completed operations (if any) and facetsubreset-specific
    # attributes and save the state
    for direction in directions:
        direction.load_state()
        direction.save_state()

    # Select subset of directions to process
    target_has_own_facet = dir_parset['target_has_own_facet']
    if target_has_own_facet:
        direction_names = [d.name for d in directions]
        target = directions[direction_names.index('target')]
    if dir_parset['ndir_process'] is not None:
        if dir_parset['ndir_process'] < len(directions):
            directions = directions[:dir_parset['ndir_process']]

            # Make sure target is still included
            direction_names = [d.name for d in directions]
            if target_has_own_facet and 'target' not in direction_names:
                directions.append(target)

            # Warn user if reimaging is to be done but they are not processing
            # the full field
            if parset['imaging_specific']['reimage_selfcaled']:
                log.warn("The reimage_selfcaled parameter is True but all directions "
                    "will not be processed. If you're interested in only a single "
                    "target in the last facet, then re-imaging will not improve results.")

    # Set various direction attributes
    for i, direction in enumerate(directions):
        # Set direction sky model
        direction.set_skymodel(initial_skymodel.copy())

        # Set peeling flag (i.e., facet calibrator should be peeled before facet
        # is imaged)
        total_flux_jy, peak_flux_jy_bm = direction.get_cal_fluxes()
        effective_flux_jy = peak_flux_jy_bm * (total_flux_jy / peak_flux_jy_bm)**0.667
        if (effective_flux_jy > parset['calibration_specific']['peel_flux_jy'] or
            direction.is_outlier):
            direction.find_peel_skymodel()
            if direction.peel_skymodel is not None:
                if not direction.is_outlier:
                   direction.peel_calibrator = True
                log.info('Direction {0} will be peeled using sky model: {1}'.format(
                    direction.name, direction.peel_skymodel))
            else:
                if direction.is_outlier:
                    log.error('Direction {} was specified as an outlier source '
                    'but an appropriate sky model is not available'.format(direction.name))
                    sys.exit(1)
                else:
                    log.warning('The flux density of direction {} exceeds peel_flux_Jy '
                        'but an appropriate peeling sky model is not available. '
                        'This direction will go through normal self calibration '
                        'instead'.format(direction.name))

        # Set full correlation solve
        if effective_flux_jy > parset['calibration_specific']['solve_all_correlations_flux_jy']:
            if not parset['calibration_specific']['spline_smooth2d']:
                log.error('The option spline_smooth2d must be enabled to use '
                    'XY and YX correlations during the slow gain solve')
                sys.exit(1)
            direction.solve_all_correlations = True

        # Set skip_facet_imaging flag
        direction.skip_facet_imaging = parset['imaging_specific']['skip_facet_imaging']

        # Set field center to that of first band (all bands have the same phase
        # center)
        direction.field_ra = bands[0].ra
        direction.field_dec = bands[0].dec

        # Reset state if specified
        if direction.name in reset_directions:
            direction.do_reset = True
            if len(reset_operations) > 0:
                direction.reset_operations = reset_operations
            else:
                direction.reset_operations = (direction.completed_operations[:] +
                    direction.started_operations[:])
        else:
            direction.do_reset = False

    # Select directions to selfcal, excluding outliers and target
    if target_has_own_facet:
        # Make sure target is not a DDE calibrator and is at end of directions list
        selfcal_directions = [d for d in directions if d.name != target.name and
                              not d.is_outlier and not d.peel_calibrator]
        directions = [d for d in directions if d.name != target.name] + [target]
    else:
        selfcal_directions = [d for d in directions if not d.is_outlier and
            not d.peel_calibrator]

    if dir_parset['ndir_selfcal'] is not None:
        if dir_parset['ndir_selfcal'] <= len(selfcal_directions):
            selfcal_directions = selfcal_directions[:dir_parset['ndir_selfcal']]

    # Divide directions into groups for selfcal
    direction_groups = factor.directions.group_directions(selfcal_directions,
        n_per_grouping=dir_parset['groupings'], allow_reordering=dir_parset['allow_reordering'])

    return directions, direction_groups
Esempio n. 39
0
def _set_up_directions(parset,
                       bands,
                       dry_run=False,
                       test_run=False,
                       reset_directions=[],
                       reset_operations=[]):
    """
    Sets up directions (facets)

    Parameters
    ----------
    parset : dict
        Parset containing processing parameters
    bands : list of Band instances
        Vis data
    dry_run : bool, optional
        If True, do not run pipelines. All parsets, etc. are made as normal
    test_run : bool, optional
        If True, use test settings. These settings are for testing purposes
        only and will not produce useful results
    reset_directions : list of str, optional
        List of direction names to be reset
    reset_operations : list of str, optional
        Llist of operations to be reset

    Returns
    -------
    directions : List of Direction instances
        All directions to be used by the run() function
    direction_groups : List of lists of Direction instances
        Groups of directions to be selfcal-ed

    """
    dir_parset = parset['direction_specific']

    max_radius_deg = dir_parset['max_radius_deg']
    ref_band = bands[-1]

    if dir_parset['faceting_skymodel'] is not None:
        import lsmtool
        log.info(
            "Using {} as sky model for source avoidance and DDE calibrator "
            "selection (if desired)".format(dir_parset['faceting_skymodel']))
        initial_skymodel = lsmtool.load(dir_parset['faceting_skymodel'])
    else:
        log.info(
            "Building local sky model for source avoidance and DDE calibrator "
            "selection (if desired)...")
        initial_skymodel = factor.directions.make_initial_skymodel(ref_band)

    log.info('Setting up directions...')
    directions = _initialize_directions(parset,
                                        initial_skymodel,
                                        ref_band,
                                        max_radius_deg=max_radius_deg,
                                        dry_run=dry_run)

    # Check with user
    if parset['interactive']:
        print(
            "\nFacet and DDE calibrator regions saved. Please check that they\n"
            "are OK before continuing. You can edit the directions file and\n"
            "continue; FACTOR will pick up any changes to it. Note: if you\n"
            "choose not to continue and you let FACTOR generate the directions\n"
            "internally, you must delete the FACTOR-made directions file\n"
            "(dir_working/factor_directions.txt) before restarting if you want\n"
            "FACTOR to regenerate it\n")
        prompt = "Continue processing (y/n)? "
        answ = raw_input(prompt)
        while answ.lower() not in ['y', 'n', 'yes', 'no']:
            answ = raw_input(prompt)
        if answ.lower() in ['n', 'no']:
            log.info('Exiting...')
            sys.exit(0)
        else:
            # Continue processing, but first re-initialize the directions to
            # pick up any changes the user made to the directions file
            directions = _initialize_directions(parset,
                                                initial_skymodel,
                                                ref_band,
                                                max_radius_deg=max_radius_deg,
                                                dry_run=dry_run)

    # Warn user if they've specified a direction to reset that does not exist
    direction_names = [d.name for d in directions]
    for name in reset_directions:
        if name not in direction_names and name != 'field':
            log.warn('Direction {} was specified for resetting but does not '
                     'exist in current list of directions'.format(name))

    # Load previously completed operations (if any) and facetsubreset-specific
    # attributes and save the state
    for direction in directions:
        direction.load_state()
        direction.save_state()

    # Select subset of directions to process
    target_has_own_facet = dir_parset['target_has_own_facet']
    if target_has_own_facet:
        direction_names = [d.name for d in directions]
        target = directions[direction_names.index('target')]
    if dir_parset['ndir_process'] is not None:
        if dir_parset['ndir_process'] < len(directions):
            directions = directions[:dir_parset['ndir_process']]

            # Make sure target is still included
            direction_names = [d.name for d in directions]
            if target_has_own_facet and 'target' not in direction_names:
                directions.append(target)

    # Set various direction attributes
    for i, direction in enumerate(directions):
        # Set direction sky model
        direction.set_skymodel(initial_skymodel.copy())

        # Set peeling flag (i.e., facet calibrator should be peeled before facet
        # is imaged)
        total_flux_jy, peak_flux_jy_bm = direction.get_cal_fluxes()
        effective_flux_jy = peak_flux_jy_bm * (total_flux_jy /
                                               peak_flux_jy_bm)**0.667
        if (effective_flux_jy > parset['calibration_specific']['peel_flux_jy']
                or direction.is_outlier):
            direction.find_peel_skymodel()
            if direction.peel_skymodel is not None:
                if not direction.is_outlier:
                    direction.peel_calibrator = True
                log.info(
                    'Direction {0} will be peeled using sky model: {1}'.format(
                        direction.name, direction.peel_skymodel))
            else:
                if direction.is_outlier:
                    log.error(
                        'Direction {} was specified as an outlier source '
                        'but an appropriate sky model is not available'.format(
                            direction.name))
                    sys.exit(1)
                else:
                    log.warning(
                        'The flux density of direction {} exceeds peel_flux_Jy '
                        'but an appropriate peeling sky model is not available. '
                        'This direction will go through normal self calibration '
                        'instead'.format(direction.name))

        # Set full correlation solve
        if effective_flux_jy > parset['calibration_specific'][
                'solve_all_correlations_flux_jy']:
            if not parset['calibration_specific']['spline_smooth2d']:
                log.error('The option spline_smooth2d must be enabled to use '
                          'XY and YX correlations during the slow gain solve')
                sys.exit(1)
            direction.solve_all_correlations = True

        # Set field center to that of first band (all bands have the same phase
        # center)
        direction.field_ra = bands[0].ra
        direction.field_dec = bands[0].dec

        # Set initial name of column that contains SUBTRACTED_DATA_ALL
        if parset['use_compression']:
            # Since we compressed the input data, SUBTRACTED_DATA_ALL is now
            # the DATA column
            direction.subtracted_data_colname = 'DATA'
            direction.use_compression = True
        else:
            direction.subtracted_data_colname = 'SUBTRACTED_DATA_ALL'
            direction.use_compression = False

        # Set any flagging parameters
        direction.flag_abstime = parset['flag_abstime']
        direction.flag_baseline = parset['flag_baseline']
        direction.flag_freqrange = parset['flag_freqrange']
        direction.flag_expr = parset['flag_expr']

        # Reset state if specified
        if direction.name in reset_directions:
            direction.do_reset = True
            if len(reset_operations) > 0:
                direction.reset_operations = reset_operations
            else:
                direction.reset_operations = (
                    direction.completed_operations[:] +
                    direction.started_operations[:])
        else:
            direction.do_reset = False

    # Select directions to selfcal, excluding outliers and target
    if target_has_own_facet:
        # Make sure target is not a DDE calibrator and is at end of directions list
        selfcal_directions = [
            d for d in directions if d.name != target.name and not d.is_outlier
            and not d.peel_calibrator
        ]
        directions = [d
                      for d in directions if d.name != target.name] + [target]
    else:
        selfcal_directions = [
            d for d in directions if not d.is_outlier and not d.peel_calibrator
        ]

    if dir_parset['ndir_selfcal'] is not None:
        if dir_parset['ndir_selfcal'] <= len(selfcal_directions):
            selfcal_directions = selfcal_directions[:
                                                    dir_parset['ndir_selfcal']]

    # Divide directions into groups for selfcal
    direction_groups = factor.directions.group_directions(
        selfcal_directions,
        n_per_grouping=dir_parset['groupings'],
        allow_reordering=dir_parset['allow_reordering'])

    return directions, direction_groups