def main(inmodel1, inmodel2, outmodel, match_by='name', radius=0.0, keep='all'):
"""
Creates mosaic
Parameters
----------
inmodel1 : str
Filename of input model 1
inmodel2 : str
Filename of input model 2
outmodel : str
Filename of output model
match_by : str, optional
The match_by parameter of LSMTool
radius : float, optional
Radius in degrees for cross match
keep : str, optional
The keep parameter of LSMTool
"""
s1 = lsmtool.load(inmodel1)
s2 = lsmtool.load(inmodel2)
s1.concatenate(s2, matchBy=match_by, radius=float(radius), keep=keep,
inheritPatches=True)
s1.group('every')
s1.write(fileName=outmodel, clobber=True)
def main(model1, model2, skymodel):
"""
Combines makesourcedb sky models
Parameters
----------
model1 : str
Filename of the input makesourcedb sky model 1
model2 : str
Filename of the input makesourcedb sky model 2
skymodel : str
Filename of the output makesourcedb sky model
"""
try:
s1 = lsmtool.load(model1)
except:
# If first sky model is empty or cannot be loaded, just copy second one
# to output file
os.system('cp -f {0} {1}'.format(model2, skymodel))
return
# Now try to load second sky model and combine with first one
try:
s2 = lsmtool.load(model2)
# Combine sky models, keeping all sources
s1.ungroup()
s2.ungroup()
s1.concatenate(s2, keep='all')
except:
# If second sky model is empty or cannot be loaded, just save s1 to output
pass
s1.write(skymodel, clobber=True)
def main(inmodel1,
inmodel2,
outmodel,
match_by='name',
radius=0.0,
keep='all'):
"""
Creates mosaic
Parameters
----------
inmodel1 : str
Filename of input model 1
inmodel2 : str
Filename of input model 2
outmodel : str
Filename of output model
match_by : str, optional
The match_by parameter of LSMTool
radius : float, optional
Radius in degrees for cross match
keep : str, optional
The keep parameter of LSMTool
"""
s1 = lsmtool.load(inmodel1)
s2 = lsmtool.load(inmodel2)
s1.concatenate(s2,
matchBy=match_by,
radius=float(radius),
keep=keep,
inheritPatches=True)
s1.group('every')
s1.write(fileName=outmodel, clobber=True)
def selectCC(self, checkBeam=True, keepInBeam=True, maskname=None):
"""
remove cc from a skymodel according to masks
checkBeam: remove according to beam (see keepInBeam)
keepInBeam: if beamReg is present and is True: remove sources outside beam
if beamReg is present and is False: remove source inside beam
maskname: a possible mask, otherwise try to use standard
"""
if maskname is None: maskname = self.maskname
if not os.path.exists(maskname):
raise("Missing mask in selectCC: %s." % maskname)
if checkBeam:
if self.beamReg is None:
raise('Missing beam in selectCC.')
logger.info('Predict (apply beam reg %s)...' % self.beamReg)
blank_image_reg(maskname, self.beamReg, inverse=keepInBeam, blankval=0) # if keep_in_beam set to 0 everything outside beam.reg
# apply mask
logger.info('%s: Apply mask (%s) on skymodel...' % (self.imagename,maskname))
lsm = lsmtool.load(self.skymodel)
lsm.select('%s == True' % maskname)
lsm.group('single') # group to 1 patch
lsm.write(self.skymodel_cut, format = 'makesourcedb', clobber=True)
del lsm
# convert from txt to blob
logger.info('%s: Make skydb...' % self.imagename)
lib_util.check_rm(self.skydb)
os.system('makesourcedb outtype="blob" format="<" in="'+self.skymodel_cut+'" out="'+self.skydb+'"')
def main(fullskymodel,
outmodel,
cal_only=False,
vertices=None,
facet_ra=0.0,
facet_dec=0.0,
cal_radius_deg=0.0):
print 'Running {0} on input data {1}'.format(__file__, str(fullskymodel))
s = lsmtool.load(fullskymodel)
if cal_only:
# Get calibrator model
dist = s.getDistance(facet_ra, facet_dec)
s.select(dist < cal_radius_deg)
else:
# Get all facet sources
x, y, midRA, midDec = s._getXY()
xv, yv = radec2xy(vertices[0], vertices[1], midRA, midDec)
xyvertices = array([[xp, yp] for xp, yp in zip(xv, yv)])
bbPath = mplPath.Path(xyvertices)
inside = zeros(len(s), dtype=bool)
for i in range(len(s)):
inside[i] = bbPath.contains_point((x[i], y[i]))
s.select(inside, force=True)
if len(s) == 0:
print('No sources found for this facet')
else:
s.write(outmodel, clobber=True)
def make_initial_skymodel(band):
"""
Makes the initial skymodel used to adjust facet edges
Parameters
----------
band : Band object
Band to use for sky model generation
Returns
-------
s : LSMTool Skymodel object
Resulting sky model
"""
import lsmtool
# Set LSMTool logging level
lsmtool._logging.setLevel('debug')
# Load sky model
s = lsmtool.load(band.skymodel_dirindep)
# Group clean components by thresholding after convolving model with
# 1-arcmin beam
s.group('threshold', FWHM='60.0 arcsec', root='facet')
s.remove('Patch = patch_*', force=True) # Remove sources that did not threshold
if len(s) == 0:
log.critical("No sources found through thresholding.")
sys.exit(1)
log.info('Found {0} sources through thresholding'.format(
len(s.getPatchNames())))
return s
def selectCC(self, keepInBeam=True):
"""
remove cc from a skymodel according to masks
keepInBeam: if beamReg is present and is True: remove sources outside beam
if beamReg is present and is False: remove source inside beam
"""
self.makeMask()
if self.facetReg is not None:
logger.info('Predict (apply facet reg %s)...' % self.facetReg)
blank_image_reg(self.maskname, self.facetReg, inverse=True, blankval=0) # set to 0 pixels outside facet mask
if self.beamReg is not None:
logger.info('Predict (apply beam reg %s)...' % self.beamReg)
blank_image_reg(self.maskname, self.beamReg, inverse=keepInBeam, blankval=0) # if keep_in_beam set to 0 everything outside beam.reg
# apply mask
logger.info('%s: Apply mask on skymodel...' % self.imagename)
lsm = lsmtool.load(self.skymodel)
lsm.select('%s == True' % self.maskname)
lsm.group('single') # group to 1 patch
lsm.write(self.skymodel_cut, format = 'makesourcedb', clobber=True)
del lsm
# convert from txt to blob
logger.info('%s: Make skydb...' % self.imagename)
lib_util.check_rm(self.skydb)
os.system('makesourcedb outtype="blob" format="<" in="'+self.skymodel_cut+'" out="'+self.skydb+'"')
def read_skymodel(skymodel, sagecalsky, sagecalcluster, admm_rho='base.rho'):
outsky = open(sagecalsky, 'w+')
outcluster = open(sagecalcluster, 'w+')
outrho = open(admm_rho, 'w+')
outsky.write('## LSM file\n')
outsky.write(
"### Name | RA (hr,min,sec) | DEC (deg,min,sec) | I | Q | U | V | SI0 | SI1 | SI2 | RM | eX | eY | eP | freq0\n"
)
outcluster.write('### Cluster file\n')
s = lsmtool.load(skymodel)
patches = s.getPatchNames()
cluster_id = 2 # cluster 1 is the target
for patch in patches:
s = lsmtool.load(skymodel)
s.select('Patch == ' + patch)
t = s.table
ra = ra_to_rad(np.array(t['Ra']))
dec = dec_to_rad(np.array(t['Dec']))
stype = [x.encode('ascii') for x in np.array(t['Type'])]
f0 = np.array(t['ReferenceFrequency'])
sI = np.array(t['I'])
SpecI = np.array(t['SpectralIndex'][:, 0])
major = asec_to_rad(np.array(t['MajorAxis']))
minor = asec_to_rad(np.array(t['MinorAxis']))
pa = math.pi / 2 - (math.pi - dec_to_rad(np.array(t['Orientation'])))
outcluster.write(str(cluster_id) + ' 1')
outrho.write(str(cluster_id) + ' 1 1.0\n')
for ci in range(ra.size):
hh, mm, ss = radToRA(ra[ci])
dd, dmm, dss = radToDec(dec[ci])
if stype[ci].decode('ascii') == 'GAUSSIAN':
name = 'G' + patch + str(ci)
else:
name = 'P' + patch + str(ci)
outsky.write(name + ' ' + str(hh) + ' ' + str(mm) + ' ' + str(ss) +
' ' + str(dd) + ' ' + str(dmm) + ' ' + str(dss) +
' ' + str(sI[ci]) + ' 0 0 0' + ' ' + str(SpecI[ci]) +
' 0 0 0' + ' ' + str(0.5 * major[ci]) + ' ' +
str(0.5 * minor[ci]) + ' ' + str(pa[ci]) + ' ' +
str(f0[ci]) + '\n')
outcluster.write(' ' + name)
outcluster.write('\n')
cluster_id += 1
outsky.close()
outcluster.close()
outrho.close()
def test_compare():
print('Compare to concat.sky')
if os.path.exists('tests/flux_ratio_vs_distance.sky'):
os.remove('tests/flux_ratio_vs_distance.sky')
c = lsmtool.load('tests/concat.sky')
c.ungroup()
c.select('I > 5.0 Jy')
s.ungroup()
s.compare(c, outDir='tests/')
assert os.path.exists('tests/flux_ratio_vs_distance.pdf')
def test_compare():
print('Compare to concat.sky')
if os.path.exists('tests/flux_ratio_vs_distance.sky'):
os.remove('tests/flux_ratio_vs_distance.sky')
c = lsmtool.load('tests/concat.sky')
c.ungroup()
c.select('I > 5.0 Jy')
s.ungroup()
s.compare(c, outDir='tests/')
assert os.path.exists('tests/flux_ratio_vs_distance.pdf')
def ft_model_cc(mss,
imagename,
c,
user_mask=None,
keep_in_beam=True,
model_column='MODEL_DATA'):
"""
skymodel : cc-list made by wsclean
keep_in_beam : if True remove everything outside primary beam, otherwise everything inside
"""
logger.info('Predict with CC...')
maskname = imagename + '-mask.fits'
skymodel = imagename + '-sources.txt'
skymodel_cut = imagename + '-sources-cut.txt'
skydb = imagename + '-sources.skydb'
# prepare mask
if not os.path.exists(maskname):
logger.info('Predict (make mask)...')
make_mask(image_name=imagename + '-MFS-image.fits',
mask_name=maskname,
threshisl=5,
atrous_do=True)
if user_mask is not None:
blank_image_reg(maskname, user_mask, inverse=False,
blankval=1) # set to 1 pixels into user_mask
blank_image_reg(
maskname, 'self/beam.reg', inverse=keep_in_beam,
blankval=0) # if keep_in_beam set to 0 everything outside beam.reg
# apply mask
logger.info('Predict (apply mask)...')
lsm = lsmtool.load(skymodel)
lsm.select('%s == True' % maskname)
fluxes = lsm.getColValues('I')
#lsm.remove(np.abs(fluxes) < 5e-4) # TEST
lsm.write(skymodel_cut, format='makesourcedb', clobber=True)
del lsm
# convert to skydb
logger.info('Predict (makesourcedb)...')
check_rm(skydb)
s.add('makesourcedb outtype="blob" format="<" in="' + skymodel_cut +
'" out="' + skydb + '"',
log='makesourcedb-c' + str(c) + '.log',
cmd_type='general')
s.run(check=True)
# predict
logger.info('Predict (ft)...')
for ms in mss:
s.add('NDPPP '+parset_dir+'/NDPPP-predict.parset msin='+ms+' msout.datacolumn='+model_column+' pre.usebeammodel=false pre.sourcedb='+skydb, \
log=ms+'_pre-c'+str(c)+'.log', cmd_type='NDPPP')
s.run(check=True)
def check_skymodel(skymodel, ra, dec, max_separation_arcmin=1.0):
"""
Searches for an appropriate sky model
"""
s = lsmtool.load(skymodel)
dist_deg = s.getDistance(ra, dec)
if any(dist_deg * 60.0 < max_separation_arcmin):
patch_position = int(
numpy.where(dist_deg * 60 < max_separation_arcmin)[0][0])
patch_name = s.getPatchNames()[patch_position]
return (True, patch_name)
else:
return (False, '')
def make_initial_skymodel(band, max_radius_deg=None):
"""
Makes the initial skymodel used to adjust facet edges
Parameters
----------
band : Band object
Band to use for sky model generation
Returns
-------
s : LSMTool Skymodel object
Resulting sky model
max_radius_deg : float, optional
Maximum radius in degrees from the phase center within which to include
sources. If None, it is set to the FWHM (i.e., a diameter of 2 * FWHM)
"""
import lsmtool
# Set LSMTool logging level
lsmtool._logging.setLevel('debug')
# Load sky model
s = lsmtool.load(band.skymodel_dirindep)
# Group clean components by thresholding after convolving model with
# 1-arcmin beam
s.group('threshold', FWHM='60.0 arcsec', root='facet')
s.remove('Patch = patch_*', force=True) # Remove sources that did not threshold
if len(s) == 0:
log.critical("No sources found through thresholding.")
sys.exit(1)
log.info('Found {0} sources through thresholding'.format(
len(s.getPatchNames())))
# Filter out sources that lie outside of maximum specific radius from phase
# center
if not hasattr(band, 'fwhm_deg'):
band.set_image_sizes()
if max_radius_deg is None:
max_radius_deg = band.fwhm_deg # means a diameter of 2 * FWHM
log.info('Removing sources beyond a radius of {0} degrees (corresponding to '
'a diameter of {1} * FWHM of the primary beam at {2} MHz)...'.format(
max_radius_deg, round(2.0*max_radius_deg/band.fwhm_deg, 1), band.freq/1e6))
dist = s.getDistance(band.ra, band.dec, byPatch=True)
s.remove(dist > max_radius_deg, aggregate=True)
return s
def cut_skymodel(skymodel_in, skymodel_out, d, do_skydb=True, do_regions=False):
"""
Load full skymodel and extract sources in the square around the calibrator of the given size
"""
lsm = lsmtool.load(skymodel_in)
# select all sources within a sqare of patch size
lsm.select('Ra > %f' % (d.position[0]-(d.size/2)/np.cos(d.position[1]* np.pi / 180.)))
lsm.select('Ra < %f' % (d.position[0]+(d.size/2)/np.cos(d.position[1]* np.pi / 180.)))
lsm.select('Dec > %f' % (d.position[1]-d.size/2))
lsm.select('Dec < %f' % (d.position[1]+d.size/2))
if do_regions: lsm.write('ddcal/masks/regions-c%02i/%s.reg' % (cmaj,d.name), format='ds9', clobber=True)
lsm.write(dir_skymodel, format='makesourcedb', clobber=True)
lib_util.check_rm(dir_skydb)
s.add('makesourcedb outtype="blob" format="<" in="%s" out="%s"' % (dir_skymodel, dir_skydb), log='makesourcedb_cl.log', commandType='general' )
s.run(check=True)
def main(model, skymodel):
"""
Groups a sky model into a single patch
Parameters
----------
model : str
Filename of the input makesourcedb sky model
skymodel : str
Filename of the output makesourcedb sky model
"""
s = lsmtool.load(model)
s.group('single')
s.write(skymodel, clobber=True)
def select_cc(self):
"""
remove cc from a skymodel according to masks
"""
self.make_mask()
if self.region_facet is not None:
logger.info('Predict (apply facet mask %s)...' % self.region_facet)
blank_image_reg(self.maskname, self.region_facet, inverse=True, blankval=0) # set to 0 pixels outside facet mask
# apply mask
logger.info('%s: Apply mask on skymodel...' % self.imagename)
lsm = lsmtool.load(self.skymodel)
lsm.select('%s == True' % self.maskname)
lsm.write(self.skymodel_cut, format='makesourcedb', clobber=True)
del lsm
def main(model, skymodel):
"""
Groups a sky model into a single patch
Parameters
----------
model : str
Filename of the input makesourcedb sky model
skymodel : str
Filename of the output makesourcedb sky model
"""
s = lsmtool.load(model)
s.group('single')
s.write(skymodel, clobber=True)
def load_skymodel(self):
"""
Loads the sky model
"""
# Set logging level to suppress confusing output from lsmtool
old_level = logger.root.getEffectiveLevel()
logger.root.setLevel('WARNING')
skymodel = lsmtool.load(self.input_skymodel_filename)
if not skymodel.hasPatches:
logger.info(
'No patches present in skymodel. Assigning every source an individual patch.'
)
skymodel.group('every')
skymodel.setPatchPositions(method='mid')
logger.root.setLevel(old_level)
self.skymodel = skymodel
def make_catalogue(self):
"""
Create catalogue for this image.
"""
import bdsf
from astropy.table import Table
img_cat = self.imagefile + '.cat'
if not os.path.exists(img_cat):
bdsf_img = bdsf.process_image(self.imagefile, rms_box=(100,30), \
thresh_pix=5, thresh_isl=3, atrous_do=False, \
adaptive_rms_box=True, adaptive_thresh=100, rms_box_bright=(30,10), quiet=True)
bdsf_img.write_catalog(outfile=img_cat,
catalog_type='srl',
format='fits',
clobber=True)
bdsf_img.write_catalog(outfile=img_cat.replace(
'.cat', '.skymodel'),
catalog_type='gaul',
format='bbs',
bbs_patches='source',
clobber=True,
srcroot='src')
else:
logging.warning('%s already exists, using it.' % img_cat)
cat = Table.read(img_cat)
# remove extended sources
extended_src = (cat['Peak_flux'] /
cat['Total_flux']) < 0.1 # ~extended source
extended_src[cat['S_Code'] == 'M'] = True # multiple-gaussian source
extended_src[cat['S_Code'] ==
'C'] = True # one gaussian + other sources island
# remove same sources from skymodel
cat_lsm = lsm.load(img_cat.replace('.cat', '.skymodel'))
for srcid in cat[extended_src]['Source_id']:
cat_lsm.remove(f'Patch == src_patch_s{srcid}')
cat.remove_rows(np.argwhere(extended_src))
self.cat = cat
self.cat_lsm = cat_lsm
logging.debug(
'%s: Number of sources detected: %i; removed %i extended sources.'
% (self.imagefile, len(self.cat), sum(extended_src)))
def find_peel_skymodel(self):
"""
Searches for an appropriate sky model for peeling
"""
if self.peel_skymodel is not None:
return
max_separation_arcmin = 1.0
factor_lib_dir = os.path.dirname(os.path.abspath(__file__))
skymodel_dir = os.path.join(os.path.split(factor_lib_dir)[0], 'skymodels')
skymodels = glob.glob(os.path.join(skymodel_dir, '*.skymodel'))
for skymodel in skymodels:
try:
s = lsmtool.load(skymodel)
dist_deg = s.getDistance(self.ra, self.dec)
if any(dist_deg*60.0 < max_separation_arcmin):
self.peel_skymodel = skymodel
break
except IOError:
pass
def main(fullskymodel, outmodel, vertices_file, cal_only=False):
"""
Makes a makesourcedb sky model for components inside input polygon
Parameters
----------
fullskymodel : str
Filename of makesourcedb sky model file containing the full-field model
outmodel : str
Filename of output sky model
vertices_file : str
Filename of pickled file with direction vertices that define polygon
cal_only : bool, optional
If True, only components wihtin cal_radius_deg of (facet_ra, facet_dec)
are selected.
"""
if type(cal_only) is str:
if cal_only.lower() == 'true':
cal_only = True
else:
cal_only = False
s = lsmtool.load(fullskymodel)
vertices = read_vertices(vertices_file, cal_only=cal_only)
# Select sources inside poly defined by vertices
x, y, midRA, midDec = s._getXY()
xv, yv = radec2xy(vertices[0], vertices[1], midRA, midDec)
xyvertices = array([[xp, yp] for xp, yp in zip(xv, yv)])
bbPath = mplPath.Path(xyvertices)
inside = zeros(len(s), dtype=bool)
for i in range(len(s)):
inside[i] = bbPath.contains_point((x[i], y[i]))
s.select(inside, force=True)
if len(s) == 0:
print('No sources found for this facet')
os.system('touch {0}'.format(outmodel))
else:
s.write(outmodel, clobber=True)
def main(fullskymodel, outmodel, cal_only=False, vertices=None, facet_ra=0.0, facet_dec=0.0, cal_radius_deg=0.0):
print 'Running {0} on input data {1}'.format(__file__, str(fullskymodel))
s = lsmtool.load(fullskymodel)
if cal_only:
# Get calibrator model
dist = s.getDistance(facet_ra, facet_dec)
s.select(dist < cal_radius_deg)
else:
# Get all facet sources
x, y, midRA, midDec = s._getXY()
xv, yv = radec2xy(vertices[0], vertices[1], midRA, midDec)
xyvertices = array([[xp, yp] for xp, yp in zip(xv, yv)])
bbPath = mplPath.Path(xyvertices)
inside = zeros(len(s), dtype=bool)
for i in range(len(s)):
inside[i] = bbPath.contains_point((x[i], y[i]))
s.select(inside, force=True)
if len(s) == 0:
print('No sources found for this facet')
else:
s.write(outmodel, clobber=True)
def main(model_root,
ms_file,
skymodel,
fits_mask=None,
min_peak_flux_jy=0.0001,
max_residual_jy=0.0):
"""
Make a makesourcedb sky model for input MS from WSClean fits model images
Parameters
----------
model_root : str
Root name of WSClean polynomial model sky model
ms_file : str
Filename of MS for which sky model is to be made. Can be a list of files
(e.g., '[ms1,ms2,...]', in which case they should all have the same
frequency
skymodel : str
Filename of the output makesourcedb sky model
fits_mask : str, optional
Filename of fits mask
min_peak_flux_jy : float, optional
Minimum absolute value of flux in Jy of a source in lowest-frequency model image
to include in output model
max_residual_jy : float, optional
Maximum acceptible total residual absolute flux in Jy
"""
min_peak_flux_jy = float(min_peak_flux_jy)
max_residual_jy = float(max_residual_jy)
if type(fits_mask) is str:
if fits_mask.lower() == 'none':
fits_mask = None
# Read MS file and get the frequency info
if '[' in ms_file and ']' in ms_file:
files = ms_file.strip('[]').split(',')
files = [f.strip() for f in files]
ms_file = files[0]
sw = pt.table(ms_file + '::SPECTRAL_WINDOW', ack=False)
ms_freq = sw.col('REF_FREQUENCY')[0]
sw.close()
# Read in sky model
polymodel = model_root + '-sources.txt'
s = lsmtool.load(polymodel)
ref_freq = float(s.getColValues('ReferenceFrequency')[0]) # Hz
# Find model images and read in frequencies
fits_models = glob.glob(model_root + '-00*-model.fits')
if len(fits_models) == 0:
# No channels images found, so look for non-MFS images
fits_models = glob.glob(model_root + '-model.fits')
if len(fits_models) == 0:
print('ERROR: no model images found')
sys.exit(1)
freqs = []
for f in fits_models:
# Get the frequency info
hdr = fits.getheader(f, 0, ignore_missing_end=True)
freqs.append(hdr['CRVAL3']) # Hz
# Determine nearest model frequency to MS frequency. We will use this to
# get the fluxes (to match the frequency blocks used during imaging and
# calibration)
sky_freq = min(freqs, key=lambda x: abs(x - ms_freq))
# Check if fits mask is empty
if fits_mask is not None:
if fits_mask.lower() == 'empty':
# Handle case in which no sources were found during masking
s.remove(np.array(range(len(s))))
mask = None
else:
mask = fits.getdata(fits_mask, 0, ignore_missing_end=True)
hdr = fits.getheader(fits_mask, 0, ignore_missing_end=True)
w = wcs.WCS(hdr)
else:
mask = None
# Discard components not in the mask, if given
if mask is not None:
pix = w.wcs_world2pix(
np.array([
s.getColValues('RA'),
s.getColValues('Dec'), [0] * len(s), [0] * len(s)
]).T, 0)
not_in_mask = []
for i, p in enumerate(pix):
if mask[0, 0, int(round(p[1])), int(round(p[0]))] < 1:
not_in_mask.append(i)
s.remove(np.array(not_in_mask))
# Set fluxes and ref frequency
specterms = s.getColValues('SpectralIndex')
stokesI = s.getColValues('I')
fluxes = []
for i in range(len(s)):
# Find flux as follows:
# flux(nu) = stokesI + term0 (nu/refnu - 1) + term1 (nu/refnu - 1)^2 + ...,
# where nu = sky_freq and refnu = ref_freq
polyterms = specterms[i].tolist()
polyterms.insert(0, stokesI[i])
fluxes.append(polyval(sky_freq / ref_freq - 1.0, polyterms))
s.setColValues('I', fluxes)
s.setColValues('ReferenceFrequency', [ms_freq] * len(s))
# Remove spectral index values
s.table.remove_column('SpectralIndex')
s.table.remove_column('LogarithmicSI')
# Write the new sky model
s.write(fileName=skymodel, clobber=True)
# set image size
imgsizepix = int(2.1 * MSs.getListObj()[0].getFWHM(freq='mid') * 3600 / 10.)
if imgsizepix % 2 != 0: imgsizepix += 1 # prevent odd img sizes
#################################################################
# Get online model
if sourcedb == '':
if not os.path.exists('tgts.skydb'):
fwhm = MSs.getListObj()[0].getFWHM(freq='min')
radeg = phasecentre[0]
decdeg = phasecentre[1]
# get model the size of the image (radius=fwhm/2)
os.system(
'wget -O tgts.skymodel "https://lcs165.lofar.eu/cgi-bin/gsmv1.cgi?coord=%f,%f&radius=%f&unit=deg"'
% (radeg, decdeg, fwhm / 2.)) # ASTRON
lsm = lsmtool.load('tgts.skymodel') #, beamMS=MSs.getListObj()[0])
lsm.remove('I<1')
lsm.write('tgts.skymodel', clobber=True)
os.system(
'makesourcedb outtype="blob" format="<" in=tgts.skymodel out=tgts.skydb'
)
apparent = False
sourcedb = 'tgts.skydb'
#################################################################################################
# Add model to MODEL_DATA
# copy sourcedb into each MS to prevent concurrent access from multiprocessing to the sourcedb
sourcedb_basename = sourcedb.split('/')[-1]
for MS in MSs.getListStr():
lib_util.check_rm(MS + '/' + sourcedb_basename)
#if c == 0:
# clean('init', MSs, size=(fwhm*1.5,fwhm*1.5), res='normal')
###
### group into patches corresponding to the mask islands
mask_cl = mosaic_image.imagename.replace('image.fits', 'mask-cl.fits')
# this mask is with no user region, done to isolate only bight compact sources
if not os.path.exists(mask_cl):
mosaic_image.beamReg = 'ddcal/beam.reg'
mosaic_image.makeMask(threshpix=7,
atrous_do=False,
remove_extended_cutoff=0.001,
maskname=mask_cl,
only_beam=True)
lsm = lsmtool.load(mosaic_image.skymodel_cut)
lsm.group(mask_cl, root='Isl')
# this removes all sources not in the mask-cl
lsm.select('Patch = Isl.*', useRegEx=True)
# this regroup sources
x = lsm.getColValues('RA', aggregate='wmean')
y = lsm.getColValues('Dec', aggregate='wmean')
flux = lsm.getColValues('I', aggregate='sum')
grouper = lib_dd_parallel.Grouper(list(zip(x, y)),
flux,
look_distance=0.3,
kernel_size=0.1,
grouping_distance=0.05)
grouper.run()
clusters = grouper.grouping()
grouper.plot()
phasecentre = MSs.getListObj()[0].getPhaseCentre()
MSs.getListObj()[0].makeBeamReg('self/beam.reg')
beamReg = 'self/beam.reg'
#################################################################
# Get online model
if sourcedb is None:
if not os.path.exists('tgts.skydb'):
fwhm = MSs.getListObj()[0].getFWHM()
radeg = phasecentre[0]
decdeg = phasecentre[1]
# get model the size of the image (radius=fwhm/2)
os.system(
'wget -O tgts.skymodel "http://172.104.228.177/cgi-bin/gsmv1.cgi?coord=%f,%f&radius=%f"'
% (radeg, decdeg, fwhm))
lsm = lsmtool.load('tgts.skymodel') #, beamMS=MSs.getListObj()[0])
#Reduces the flux of clean component according to a primary beam function
#NOTE: monochromatic approximation!
center = SkyCoord(phasecentre[0] * u.deg, phasecentre[1] * u.deg)
sources = SkyCoord(
lsm.getColValues('RA') * u.deg,
lsm.getColValues('Dec') * u.deg)
d = center.separation(sources)
# from http://www.aips.nrao.edu/cgi-bin/ZXHLP2.PL?PBCOR (converto to arcmin and multiply by freq in GHz)
d = d.deg * 60 * np.mean(MSs.getListObj()[0].getFreqs()) / 1.e9
I = lsm.getColValues('I')
parm = [-3.397, 47.192, -30.931, 7.803] # 325 MHz GMRT
I_corr = I * (1 + (parm[0]/10**3)*d**2 + (parm[1]/10**7)*d**4 + \
(parm[2]/10**10)*d**6 + (parm[3]/10**13)*d**8)
lsm.setColValues('I', I_corr)
lsm.write('tgts.skymodel', clobber=True)
def main(ms_input, SkymodelPath, Radius="5.", DoDownload="True", Source="TGSS"):
"""
Download the skymodel for the target field
Parameters
----------
ms_input : str
String from the list (map) of the target MSs
SkymodelPath : str
Full name (with path) to the skymodel; if YES is true, the skymodel will be downloaded here
Radius : string with float (default = "5.")
Radius for the TGSS/GSM cone search in degrees
DoDownload : str ("Force" or "True" or "False")
Download or not the TGSS skymodel or GSM.
"Force": download skymodel from TGSS or GSM, delete existing skymodel if needed.
"True" or "Yes": use existing skymodel file if it exists, download skymodel from
TGSS or GSM if it does not.
"False" or "No": Do not download skymodel, raise an exception if skymodel
file does not exist.
"""
FileExists = os.path.isfile(SkymodelPath)
if (not FileExists and os.path.exists(SkymodelPath)):
raise ValueError("download_tgss_skymodel_target: Path: \"%s\" exists but is not a file!"%(SkymodelPath))
download_flag = False
if not os.path.exists(os.path.dirname(SkymodelPath)):
os.makedirs(os.path.dirname(SkymodelPath))
if DoDownload.upper() == "FORCE":
if FileExists:
os.remove(SkymodelPath)
download_flag = True
elif DoDownload.upper() == "TRUE" or DoDownload.upper() == "YES":
if FileExists:
print("USING the exising skymodel in "+ SkymodelPath)
return(0)
else:
download_flag = True
elif DoDownload.upper() == "FALSE" or DoDownload.upper() == "NO":
if FileExists:
print("USING the exising skymodel in "+ SkymodelPath)
return(0)
else:
raise ValueError("download_tgss_skymodel_target: Path: \"%s\" does not exist and skymodel download is disabled!"%(SkymodelPath))
# If we got here, then we are supposed to download the skymodel.
assert download_flag is True # Jaja, belts and suspenders...
print("DOWNLOADING skymodel for the target into "+ SkymodelPath)
# Reading a MS to find the coordinate (pyrap)
[RATar,DECTar]=grab_coord_MS(input2strlist_nomapfile(ms_input)[0])
# Downloading the skymodel, skip after five tries
errorcode = 1
tries = 0
while errorcode != 0 and tries < 5:
if Source == 'TGSS':
errorcode = os.system("wget -O "+SkymodelPath+ " \'http://tgssadr.strw.leidenuniv.nl/cgi-bin/gsmv4.cgi?coord="+str(RATar)+","+str(DECTar)+"&radius="+str(Radius)+"&unit=deg&deconv=y\' ")
elif Source == 'GSM':
errorcode = os.system("wget -O "+SkymodelPath+ " \'https://lcs165.lofar.eu/cgi-bin/gsmv1.cgi?coord="+str(RATar)+","+str(DECTar)+"&radius="+str(Radius)+"&unit=deg&deconv=y\' ")
time.sleep(5)
tries += 1
if not os.path.isfile(SkymodelPath):
raise IOError("download_tgss_skymodel_target: Path: \"%s\" does not exist after trying to download the skymodel."%(SkymodelPath))
# Treat all sources as one group (direction)
skymodel = lsmtool.load(SkymodelPath)
skymodel.group('single')
skymodel.write(clobber=True)
return(0)
def selfCalRunFuncStatistics(self):
############################################################
# Extract Statistics from D.Rafferty LSMtool Module
############################################################
# Create stats directory without flux cut
plotsGSMDir_noFluxCut = self.statDir+'LSMTools/GSM_Comparison_noFluxCut/'
plotNVSSDir_noFluxCut = self.statDir+'LSMTools/NVSS_Comparison_noFluxCut/'
cmd1 = 'mkdir -p %s'%(plotsGSMDir_noFluxCut)
cmd2 = 'mkdir -p %s'%(plotNVSSDir_noFluxCut)
os.system(cmd1)
os.system(cmd2)
if self.i != 0:
plotPreviousIterDir_noFluxCut = self.statDir+'LSMTools/Iter%s_Iter%s_Comparison_noFluxCut/'%(self.i-1,self.i)
cmd3 ='mkdir -p %s'%(plotPreviousIterDir_noFluxCut)
os.system(cmd3)
# Create stats directory with flux cut I>500 mJy
plotsGSMDir_500mJyCut = self.statDir+'LSMTools/GSM_Comparison_500mJyCut/'
plotNVSSDir_500mJyCut = self.statDir+'LSMTools/NVSS_Comparison_500mJyCut/'
cmd1 = 'mkdir -p %s'%(plotsGSMDir_500mJyCut)
cmd2 = 'mkdir -p %s'%(plotNVSSDir_500mJyCut)
os.system(cmd1)
os.system(cmd2)
if self.i != 0:
plotPreviousIterDir_500mJyCut = self.statDir+'LSMTools/Iter%s_Iter%s_Comparison_500mJyCut/'%(self.i-1,self.i)
cmd3 ='mkdir -p %s'%(plotPreviousIterDir_500mJyCut)
os.system(cmd3)
# Load Skymodels
pybdsm_model_current = lsmtool.load(self.statisticsSkymodelCurrent)
if self.i != 0:
pybdsm_model_previous = lsmtool.load(self.statisticsSkymodelPrevious)
gsm_model = lsmtool.load('gsm', VOPosition=[self.ra_target, self.dec_target], VORadius='%s degree'%(self.FOV))
nvss_model = lsmtool.load('nvss', VOPosition=[self.ra_target, self.dec_target], VORadius='%s degree'%(self.FOV))
# Run comparison no flux cut
pybdsm_model_current.compare(gsm_model, outDir='%s'%(plotsGSMDir_noFluxCut), radius='2 arcmin')
pybdsm_model_current.compare(nvss_model, outDir='%s'%(plotNVSSDir_noFluxCut), radius='2 arcmin')
if self.i != 0:
pybdsm_model_current.compare(pybdsm_model_previous, outDir='%s'%(plotPreviousIterDir_noFluxCut), radius='2 arcmin')
# Run comparison with flux cut I>500mJy
pybdsm_model_current.select('I > 500 mJy')
if self.i != 0:
pybdsm_model_previous.select('I > 500 mJy')
pybdsm_model_current.compare(gsm_model, outDir='%s'%(plotsGSMDir_500mJyCut), radius='2 arcmin', excludeByFlux=True)
pybdsm_model_current.compare(nvss_model, outDir='%s'%(plotNVSSDir_500mJyCut), radius='2 arcmin', excludeByFlux=True)
if self.i != 0:
pybdsm_model_current.compare(pybdsm_model_previous, outDir='%s'%(plotPreviousIterDir_500mJyCut), radius='2 arcmin', excludeByFlux=True)
############################################################
# Extract Instrument parmdb plots from Sarrvesh Module
############################################################
# generate instruments table plots
list_instrument_MS = range(len(self.Files))
list_MS = range(len(self.Files))
for k in range(len(self.Files)):
if self.outerfovclean =='no':
list_instrument_MS[k] = """%s%s/instrument"""%(self.IterDir,self.Files[k])
list_MS[k] = """%s"""%(self.Files[k])
if self.outerfovclean =='yes':
list_instrument_MS[k] = """%s%s_sub%s/instrument"""%(self.IterDir,self.Files[k],self.preprocessIndex)
list_MS[k] = """%s_sub%s"""%(self.Files[k],self.preprocessIndex)
parmdbOutputDir = """%s/Parmdb_Plots/%s/"""%(self.statDir,list_MS[k])
cmd = """mkdir -p %s"""%(parmdbOutputDir)
os.system(cmd)
self.makePhasePlots("""%s"""%(list_instrument_MS[k]),"""%s"""%(parmdbOutputDir))
############################################################
# Other Stats extracted with pybdsm
############################################################
pybdsmStatDir = """%sPybdsm_Stats/"""%(self.statDir)
cmd_pybdsmStats = """mkdir -p %s"""%(pybdsmStatDir)
os.system(cmd_pybdsmStats)
pybdsmStatFile = """%sExtractedStats"""%(pybdsmStatDir)
filepybdsm = open(pybdsmStatFile,'w')
cmd_pybdsm = """
******************************************************************************
Informations extracted with pybdsm
- The (3-sigma) clipped rms = %s Jy
- The Mean flux on the image = %s Jy
- The Total Flux on the image = %s Jy
"""%(self.rmsclipped,self.Mean,self.TotalFlux)
filepybdsm.write(cmd_pybdsm)
filepybdsm.close()
def main(h5parmfile,
soltabname='phase000',
outroot='',
bounds_deg=None,
bounds_mid_deg=None,
skymodel=None,
solsetname='sol000',
ressoltabname='',
padding_fraction=1.4,
cellsize_deg=0.1,
smooth_deg=0,
gsize_deg=0,
time_avg_factor=1,
fasth5parm=None,
interp_kind='nearest'):
"""
Make a-term FITS images
Parameters
----------
h5parmfile : str
Filename of h5parm
soltabname : str
Soltab containing solutions or screen fit
outroot : str
Root of filename of output FITS file (root+'_0.fits')
bounds_deg : list
List of [maxRA, minDec, minRA, maxDec] for image bounds
bounds_mid_deg : list
List of [RA, Dec] for midpoint of image bounds
skymodel : str
Filename of calibration sky model (needed for patch positions)
solsetname : str, optional
Name of solset
ressoltabname : str, optional
Soltab containing the screen residuals
padding_fraction : float, optional
Fraction of total size to pad with (e.g., 0.2 => 20% padding all around)
cellsize_deg : float, optional
Cellsize of output image
smooth_deg : float, optional
Size of smoothing kernel in degrees to apply
gsize_deg : float, optional
FWHM in degrees of Gaussian to add at patch locations (to enforce that
solutions at these locations are exactly equal to those in the h5parm)
time_avg_factor : int, optional
Averaging factor in time for fast-phase corrections
fasth5parm : str, optional
Filename of fast-phase h5parm to be added together with input h5parm
(interpolation of the input h5parm is done first)
interp_kind : str, optional
Kind of interpolation to use, if fasth5parm is given. Can be any
supported by scipy.interpolate.interp1d
Returns
-------
result : dict
Dict with list of FITS files
"""
# Read in solutions
H = h5parm(h5parmfile)
solset = H.getSolset(solsetname)
if 'gain' in soltabname:
soltab = solset.getSoltab(soltabname.replace('gain', 'amplitude'))
soltab_ph = solset.getSoltab(soltabname.replace('gain', 'phase'))
else:
soltab = solset.getSoltab(soltabname)
if type(bounds_deg) is str:
bounds_deg = [
float(f.strip()) for f in bounds_deg.strip('[]').split(';')
]
if type(bounds_mid_deg) is str:
bounds_mid_deg = [
float(f.strip()) for f in bounds_mid_deg.strip('[]').split(';')
]
if padding_fraction is not None:
padding_fraction = float(padding_fraction)
padding_ra = (bounds_deg[2] - bounds_deg[0]) * (padding_fraction - 1.0)
padding_dec = (bounds_deg[3] - bounds_deg[1]) * (padding_fraction -
1.0)
bounds_deg[0] -= padding_ra
bounds_deg[1] -= padding_dec
bounds_deg[2] += padding_ra
bounds_deg[3] += padding_dec
cellsize_deg = float(cellsize_deg)
gsize_deg = float(gsize_deg)
gsize_pix = gsize_deg / cellsize_deg
smooth_deg = float(smooth_deg)
smooth_pix = smooth_deg / cellsize_deg
time_avg_factor = int(time_avg_factor)
# Do Voronoi tessellation + smoothing
if 'amplitude' in soltab.getType():
# complexgain
vals = soltab.val
vals_ph = soltab_ph.val
else:
# scalarphase -> set amplitudes to unity
vals_ph = soltab.val
vals = np.ones_like(vals_ph)
times = soltab.time
freqs = soltab.freq
ants = soltab.ant
axis_names = soltab.getAxesNames()
source_names = soltab.dir[:]
# Load fast-phase solutions if needed and combine with slow gains by interpolating
# the slow gains to the fast time grid
if 'amplitude' in soltab.getType() and fasth5parm is not None:
H_fast = h5parm(fasth5parm)
solset_fast = H_fast.getSolset('sol000')
soltab_fast = solset_fast.getSoltab('phase000')
times_fast = soltab_fast.time
freqs_fast = soltab_fast.freq
# Interpolate the slow gains to the fast times and frequencies
axis_names = soltab.getAxesNames()
time_ind = axis_names.index('time')
freq_ind = axis_names.index('freq')
if len(times) == 1:
# If just a single time, we just repeat the values as needed
fast_axis_names = soltab_fast.getAxesNames()
fast_time_ind = fast_axis_names.index('time')
fast_freq_ind = fast_axis_names.index('freq')
new_shape = list(vals.shape)
new_shape[time_ind] = soltab_fast.val.shape[fast_time_ind]
new_shape[freq_ind] = soltab_fast.val.shape[fast_freq_ind]
vals = np.resize(vals, new_shape)
vals_ph = np.resize(vals, new_shape)
else:
# Interpolate (in log space for amps)
logvals = np.log10(vals)
f = si.interp1d(times,
logvals,
axis=time_ind,
kind=interp_kind,
fill_value='extrapolate')
logvals = f(times_fast)
f = si.interp1d(freqs,
logvals,
axis=freq_ind,
kind=interp_kind,
fill_value='extrapolate')
logvals = f(freqs_fast)
vals = 10**(logvals)
f = si.interp1d(times,
vals_ph,
axis=time_ind,
kind=interp_kind,
fill_value='extrapolate')
vals_ph = f(times_fast)
f = si.interp1d(freqs,
vals_ph,
axis=freq_ind,
kind=interp_kind,
fill_value='extrapolate')
vals_ph = f(freqs_fast)
for p in range(2):
vals_ph[:, :, :, :, p] += soltab_fast.val
freqs = freqs_fast
times = times_fast
# Make blank output FITS file (type does not matter at this point)
midRA = bounds_mid_deg[0]
midDec = bounds_mid_deg[1]
temp_image = outroot + '.tmp'
imsize = (bounds_deg[3] - bounds_deg[1]) # deg
imsize = int(imsize / cellsize_deg) # pix
misc.make_template_image(temp_image,
midRA,
midDec,
ximsize=imsize,
yimsize=imsize,
cellsize_deg=cellsize_deg,
freqs=freqs,
times=[0.0],
antennas=soltab.ant,
aterm_type='tec')
hdu = pyfits.open(temp_image, memmap=False)
data = hdu[0].data
w = wcs.WCS(hdu[0].header)
RAind = w.axis_type_names.index('RA')
Decind = w.axis_type_names.index('DEC')
# Get x, y coords for directions in pixels. We use the input calibration sky
# model for this, as the patch positions written to the h5parm file by DPPP may
# be different
skymod = lsmtool.load(skymodel)
source_dict = skymod.getPatchPositions()
source_positions = []
for source in source_names:
radecpos = source_dict[source.strip('[]')]
source_positions.append([radecpos[0].value, radecpos[1].value])
source_positions = np.array(source_positions)
ra_deg = source_positions.T[0]
dec_deg = source_positions.T[1]
xy = []
for RAvert, Decvert in zip(ra_deg, dec_deg):
ra_dec = np.array([[0.0, 0.0, 0.0, 0.0, 0.0]])
ra_dec[0][RAind] = RAvert
ra_dec[0][Decind] = Decvert
xy.append((w.wcs_world2pix(ra_dec, 0)[0][RAind],
w.wcs_world2pix(ra_dec, 0)[0][Decind]))
# Get boundary for imaging region in pixels
ra_dec = np.array([[0.0, 0.0, 0.0, 0.0, 0.0]])
ra_dec[0][RAind] = bounds_deg[0]
ra_dec[0][Decind] = bounds_deg[1]
field_minxy = (w.wcs_world2pix(ra_dec,
0)[0][RAind], w.wcs_world2pix(ra_dec,
0)[0][Decind])
ra_dec[0][RAind] = bounds_deg[2]
ra_dec[0][Decind] = bounds_deg[3]
field_maxxy = (w.wcs_world2pix(ra_dec,
0)[0][RAind], w.wcs_world2pix(ra_dec,
0)[0][Decind])
# Generate array of outer points used to constrain the facets
nouter = 64
means = np.ones((nouter, 2)) * np.array(xy).mean(axis=0)
offsets = []
angles = [np.pi / (nouter / 2.0) * i for i in range(0, nouter)]
for ang in angles:
offsets.append([np.cos(ang), np.sin(ang)])
radius = 2.0 * np.sqrt((field_maxxy[0] - field_minxy[0])**2 +
(field_maxxy[1] - field_minxy[1])**2)
scale_offsets = radius * np.array(offsets)
outer_box = means + scale_offsets
# Tessellate and clip
points_all = np.vstack([xy, outer_box])
vor = Voronoi(points_all)
lines = [
shapely.geometry.LineString(vor.vertices[line])
for line in vor.ridge_vertices if -1 not in line
]
polygons = [poly for poly in shapely.ops.polygonize(lines)]
# Index polygons to directions
ind = []
for i, xypos in enumerate(xy):
for poly in polygons:
if poly.contains(Point(xypos)):
poly.index = i
# Rasterize the polygons to an array, with the value being equal to the
# polygon's index+1
data_template = np.ones(data[0, 0, 0, :, :].shape)
data_rasertize_template = np.zeros(data[0, 0, 0, :, :].shape)
for poly in polygons:
verts_xy = poly.exterior.xy
verts = []
for x, y in zip(verts_xy[0], verts_xy[1]):
verts.append((x, y))
poly_raster = misc.rasterize(verts,
data_template.copy()) * (poly.index + 1)
filled = np.where(poly_raster > 0)
data_rasertize_template[filled] = poly_raster[filled]
# Identify any duplicate times and remove
delta_times = times[1:] - times[:-1] # time at center of solution interval
nodupind = np.where(delta_times > 0.1)
times = times[nodupind]
if 'pol' in axis_names:
vals = np.squeeze(vals[nodupind, :, :, :, :])
vals_ph = np.squeeze(vals_ph[nodupind, :, :, :, :])
else:
vals = np.squeeze(vals[nodupind, :, :, :])
vals_ph = np.squeeze(vals_ph[nodupind, :, :, :])
# Identify any gaps in time (frequency gaps are not allowed), as we need to
# output a separate FITS file for each time chunk
delta_times = times[1:] - times[:-1] # time at center of solution interval
timewidth = np.min(delta_times)
gaps = np.where(delta_times > timewidth * 1.2)
gaps_ind = gaps[0] + 1
gaps_ind = np.append(gaps_ind, np.array([len(times)]))
# Add additional breaks to gaps_ind to keep memory use within that available
# From experience, making a (30, 46, 62, 4, 146, 146) aterm image needs around
# 30 GB of memory
if soltab.getType() == 'tec':
max_ntimes = 15 * 46 * 4
else:
max_ntimes = 15
# TODO: adjust max_ntimes depending on available memory and time_avg_factor
check_gaps = True
while check_gaps:
check_gaps = False
g_start = 0
gaps_ind_copy = gaps_ind.copy()
for gnum, g_stop in enumerate(gaps_ind_copy):
if g_stop - g_start > max_ntimes:
new_gap = g_start + int((g_stop - g_start) / 2)
gaps_ind = np.insert(gaps_ind, gnum, np.array([new_gap]))
check_gaps = True
break
g_start = g_stop
if soltab.getType() == 'tec':
# TEC solutions
# input data are [time, ant, dir, freq]
# output data are [RA, DEC, ANTENNA, FREQ, TIME].T
# Now loop over stations, frequencies, and times and fill in the correct
# values
outfiles = []
g_start = 0
for gnum, g_stop in enumerate(gaps_ind):
outfile = '{0}_{1}.fits'.format(outroot, gnum)
misc.make_template_image(temp_image,
midRA,
midDec,
ximsize=imsize,
yimsize=imsize,
cellsize_deg=cellsize_deg,
times=times[g_start:g_stop],
freqs=freqs,
antennas=soltab.ant,
aterm_type='tec')
hdu = pyfits.open(temp_image, memmap=False)
data = hdu[0].data
w = wcs.WCS(hdu[0].header)
for t, time in enumerate(times[g_start:g_stop]):
for f, freq in enumerate(freqs):
for s, stat in enumerate(ants):
for poly in polygons:
ind = np.where(
data_rasertize_template == poly.index + 1)
data[t, f, s, ind[0],
ind[1]] = vals[t + g_start, s, poly.index, 0]
# Smooth if desired
if smooth_pix > 0:
data[t, f, s, :, :] = ndimage.gaussian_filter(
data[t, f, s, :, :],
sigma=(smooth_pix, smooth_pix),
order=0)
# Add Gaussians at patch positions if desired
if gsize_pix > 0:
for i, (x, y) in enumerate(xy):
# Only do this if patch is inside the region of interest
if int(x) >= 0 and int(
x) < data.shape[4] and int(
y) >= 0 and int(y) < data.shape[3]:
A = vals[t + g_start, s, i,
0] - data[t, f, s,
int(y),
int(x)]
data[t, f, s, :, :] += guassian_image(
A, x, y, data.shape[4], data.shape[3],
gsize_pix)
g_start = g_stop
# Write FITS file
hdu[0].data = data
hdu.writeto(outfile, overwrite=True)
outfiles.append(outfile)
os.remove(temp_image)
outfile = open(outroot + '.txt', 'w')
outfile.writelines([o + '\n' for o in outfiles])
outfile.close()
else:
# Gain solutions
# input data are [time, freq, ant, dir, pol] for slow gains (complexgain)
# and [time, freq, ant, dir] for fast (non-tec) phases (scalarphase)
# output data are [RA, DEC, MATRIX, ANTENNA, FREQ, TIME].T
# Now loop over stations, frequencies, and times and fill in the correct
# matrix values (matrix dimension has 4 elements: real XX, imaginary XX,
# real YY and imaginary YY)
outfiles = []
g_start = 0
for gnum, g_stop in enumerate(gaps_ind):
outfile = '{0}_{1:04}.fits'.format(outroot, gnum)
misc.make_template_image(temp_image,
midRA,
midDec,
ximsize=imsize,
yimsize=imsize,
cellsize_deg=cellsize_deg,
times=times[g_start:g_stop],
freqs=freqs,
antennas=soltab.ant,
aterm_type='gain')
hdu = pyfits.open(temp_image, memmap=False)
data = hdu[0].data
w = wcs.WCS(hdu[0].header)
for t, time in enumerate(times[g_start:g_stop]):
for f, freq in enumerate(freqs):
for s, stat in enumerate(ants):
for p, poly in enumerate(polygons):
ind = np.where(
data_rasertize_template == poly.index + 1)
if 'pol' in axis_names:
val_amp_xx = vals[t + g_start, f, s,
poly.index, 0]
val_amp_yy = vals[t + g_start, f, s,
poly.index, 1]
val_phase_xx = vals_ph[t + g_start, f, s,
poly.index, 0]
val_phase_yy = vals_ph[t + g_start, f, s,
poly.index, 1]
else:
val_amp_xx = vals[t + g_start, f, s,
poly.index]
val_amp_yy = vals[t + g_start, f, s,
poly.index]
val_phase_xx = vals_ph[t + g_start, f, s,
poly.index]
val_phase_yy = vals_ph[t + g_start, f, s,
poly.index]
data[t, f, s, 0, ind[0],
ind[1]] = val_amp_xx * np.cos(val_phase_xx)
data[t, f, s, 2, ind[0],
ind[1]] = val_amp_yy * np.cos(val_phase_yy)
data[t, f, s, 1, ind[0],
ind[1]] = val_amp_xx * np.sin(val_phase_xx)
data[t, f, s, 3, ind[0],
ind[1]] = val_amp_yy * np.sin(val_phase_yy)
# Smooth if desired
if smooth_pix > 0:
data[t, f, s, :, :, :] = ndimage.gaussian_filter(
data[t, f, s, :, :, :],
sigma=(0, smooth_pix, smooth_pix),
order=0)
# Add Gaussians at patch positions if desired
if gsize_pix > 0:
for i, (x, y) in enumerate(xy):
# Only do this if patch is inside the region of interest
if int(x) >= 0 and int(
x) < data.shape[4] and int(
y) >= 0 and int(y) < data.shape[3]:
if 'pol' in axis_names:
val_amp_xx = vals[t + g_start, f, s, i,
0]
val_amp_yy = vals[t + g_start, f, s, i,
1]
val_phase_xx = vals_ph[t + g_start, f,
s, i, 0]
val_phase_yy = vals_ph[t + g_start, f,
s, i, 1]
else:
val_amp_xx = vals[t + g_start, f, s, i]
val_amp_yy = vals[t + g_start, f, s, i]
val_phase_xx = vals_ph[t + g_start, f,
s, i]
val_phase_yy = vals_ph[t + g_start, f,
s, i]
A = val_amp_xx * np.cos(
val_phase_xx) - data[t, f, s, 0,
int(y),
int(x)]
data[t, f, s, 0, :, :] += guassian_image(
A, x, y, data.shape[5], data.shape[4],
gsize_pix)
A = val_amp_yy * np.cos(
val_phase_yy) - data[t, f, s, 2,
int(y),
int(x)]
data[t, f, s, 2, :, :] += guassian_image(
A, x, y, data.shape[5], data.shape[4],
gsize_pix)
A = val_amp_xx * np.sin(
val_phase_xx) - data[t, f, s, 1,
int(y),
int(x)]
data[t, f, s, 1, :, :] += guassian_image(
A, x, y, data.shape[5], data.shape[4],
gsize_pix)
A = val_amp_yy * np.sin(
val_phase_yy) - data[t, f, s, 3,
int(y),
int(x)]
data[t, f, s, 3, :, :] += guassian_image(
A, x, y, data.shape[5], data.shape[4],
gsize_pix)
# If averaging in time, make a new template image with
# fewer times and write to that instead
if time_avg_factor > 1:
times_avg = times[g_start:g_stop:time_avg_factor]
ntimes = len(times_avg)
misc.make_template_image(temp_image + '.avg',
midRA,
midDec,
ximsize=imsize,
yimsize=imsize,
cellsize_deg=cellsize_deg,
times=times_avg,
freqs=freqs,
antennas=soltab.ant,
aterm_type='gain')
hdu = pyfits.open(temp_image + '.avg', memmap=False)
data_avg = hdu[0].data
# Average
for t, time in enumerate(times_avg):
incr = min(
time_avg_factor,
len(times[g_start:g_stop]) - t * time_avg_factor)
data_avg[t, :, :, :, :, :] = np.nanmean(
data[t:t + incr, :, :, :, :, :], axis=0)
data = data_avg
else:
ntimes = len(times[g_start:g_stop])
# Ensure there are no NaNs in the images, as WSClean will produced uncorrected,
# uncleaned images if so. We replace NaNs with 1.0 and 0.0 for real and
# imaginary parts, respectively
# Note: we iterate over time to reduce memory usage
for t in range(ntimes):
for p in range(4):
if p % 2:
# Imaginary elements
nanval = 0.0
else:
# Real elements
nanval = 1.0
data[t, :, :, p, :, :][np.isnan(data[t, :, :,
p, :, :])] = nanval
# Write FITS file
hdu[0].data = data
hdu.writeto(outfile, overwrite=True)
outfiles.append(outfile)
os.remove(temp_image)
hdu = None
data = None
# Update start time index
g_start = g_stop
outfile = open(outroot + '.txt', 'w')
outfile.writelines([o + '\n' for o in outfiles])
outfile.close()
# Runs steps for validation
import lsmtool
s = lsmtool.load('tests/no_patches.sky')
print('Select individual sources with Stokes I fluxes above 1 Jy')
s.select('I > 1.0 Jy')
print('Transfer patches from patches.sky')
s.transfer('tests/patches.sky')
print('Remove patches with total fluxes below 2 Jy')
s.remove('I < 2.0 Jy', aggregate='sum')
print('Ungroup the skymodel')
s.ungroup()
print('Concatenate with concat.sky')
s.concatenate('tests/concat.sky', matchBy = 'position', radius = '30 arcsec', keep = 'from2')
print('Compare to concat.sky')
c = lsmtool.load('tests/concat.sky')
c.ungroup()
c.select('I > 5.0 Jy')
s.ungroup()
s.compare(c, outDir='tests/')
print('Add a source')
s.add({'Name': 'src1', 'Type': 'POINT', 'Ra': 277.4232, 'Dec': 48.3689, 'I': 0.69})
model_skymodel = 'ddcal/c%02i/skymodels/%s-best-source.txt' % (
cmaj, d.name)
model_skydb = 'ddcal/c%02i/skymodels/%s-best-source.skydb' % (
cmaj, d.name)
os.system('cp %s %s' %
(d.get_model('best') + '-sources.txt', model_skymodel))
# restrict to initial mask
logger.info('Restrict model to initial region')
os.system('cp ' + d.model['best'] + '-mask.fits' + ' ' +
d.model['best'] + '-mask-restricted.fits')
lib_img.blank_image_reg(d.model['best'] + '-mask-restricted.fits',
d.get_region(),
inverse=True,
blankval=0.)
lsm = lsmtool.load(model_skymodel)
lsm.select('%s == True' %
(d.model['best'] + '-mask-restricted.fits'))
lsm.write(model_skymodel, format='makesourcedb', clobber=True)
lib_util.check_rm(model_skydb)
s.add('makesourcedb outtype="blob" format="<" in="%s" out="%s"' %
(model_skymodel, model_skydb),
log='makesourcedb_cl.log',
commandType='general')
s.run()
# remove the DD-cal from original dataset using new solutions
with w.if_todo('%s-subtract' % logstring):
# Predict - ms:MODEL_DATA
def make_skymodel(self, index):
"""
Makes predict sky model
Parameters
----------
index : int
Iteration index
"""
# First check whether sky model already exists due to a previous run and attempt
# to load it if so
dst_dir = os.path.join(self.field.working_dir, 'skymodels',
'predict_{}'.format(index))
misc.create_directory(dst_dir)
self.predict_skymodel_file = os.path.join(
dst_dir, '{}_predict_skymodel.txt'.format(self.name))
if os.path.exists(self.predict_skymodel_file):
skymodel = lsmtool.load(str(self.predict_skymodel_file))
else:
# If sky model does not already exist, make it
if self.is_outlier or self.is_bright_source:
# For outlier and bright-source sectors, we use the sky model made earlier,
# with no filtering
skymodel = self.predict_skymodel
else:
# For imaging sectors, we use the full calibration sky model and filter it
# to keep only sources inside the sector
skymodel = self.calibration_skymodel.copy()
skymodel = self.filter_skymodel(skymodel)
# Remove the bright sources from the sky model if they will be predicted and
# subtracted separately (so that they aren't subtracted twice)
if self.field.peel_bright_sources and not self.is_outlier and not self.is_bright_source:
source_names = skymodel.getColValues('Name')
bright_source_names = self.field.bright_source_skymodel.getColValues(
'Name')
matching_ind = []
for i, sn in enumerate(source_names):
if sn in bright_source_names:
matching_ind.append(i)
if len(matching_ind) > 0:
skymodel.remove(np.array(matching_ind))
# Write filtered sky model to file for later prediction
if len(skymodel) > 0:
skymodel.write(self.predict_skymodel_file, clobber=True)
else:
# No sources, so just make a dummy sky model with single,
# very faint source at center
dummylines = [
"Format = Name, Type, Patch, Ra, Dec, I, SpectralIndex, LogarithmicSI, "
"ReferenceFrequency='100000000.0', MajorAxis, MinorAxis, Orientation\n"
]
ra = misc.ra2hhmmss(self.ra)
sra = str(ra[0]).zfill(2) + ':' + str(
ra[1]).zfill(2) + ':' + str("%.6f" % (ra[2])).zfill(6)
dec = misc.dec2ddmmss(self.dec)
decsign = ('-' if dec[3] < 0 else '+')
sdec = decsign + str(dec[0]).zfill(2) + '.' + str(
dec[1]).zfill(2) + '.' + str("%.6f" % (dec[2])).zfill(6)
patch = self.calibration_skymodel.getPatchNames()[0]
dummylines.append(',,{0},{1},{2}\n'.format(patch, sra, sdec))
dummylines.append('s0c0,POINT,{0},{1},{2},0.00000001,'
'[0.0,0.0],false,100000000.0,,,\n'.format(
patch, sra, sdec))
with open(self.predict_skymodel_file, 'w') as f:
f.writelines(dummylines)
skymodel = lsmtool.load(str(self.predict_skymodel_file))
# Save list of patches (directions) in the format written by DDECal in the h5parm
self.patches = ['[{}]'.format(p) for p in skymodel.getPatchNames()]
# Find nearest patch to flux-weighted center of the sector sky model
if not self.is_outlier and not self.is_bright_source:
tmp_skymodel = skymodel.copy()
tmp_skymodel.group('single')
ra, dec = tmp_skymodel.getPatchPositions(method='wmean',
asArray=True)
patch_dist = skymodel.getDistance(ra[0], dec[0],
byPatch=True).tolist()
patch_names = skymodel.getPatchNames()
self.central_patch = patch_names[patch_dist.index(min(patch_dist))]
# Filter the field source sky model and store source sizes
all_source_names = self.field.source_skymodel.getColValues(
'Name').tolist()
source_names = skymodel.getColValues('Name')
if len(source_names
) == 1 and source_names[0] not in all_source_names:
# This occurs when a dummy sky model was made above, so skip the size
# determination below
source_skymodel = []
else:
in_sector = np.array(
[all_source_names.index(sn) for sn in source_names])
source_skymodel = self.field.source_skymodel.copy()
source_skymodel.select(in_sector)
if len(source_skymodel) > 0:
self.source_sizes = source_skymodel.getPatchSizes(
units='degree')
else:
self.source_sizes = [0.0]
# Set the parameters for predict
self.set_prediction_parameters()
# make beam to the first mid null
phasecentre = MSs.getListObj()[0].getPhaseCentre()
# MSs.getListObj()[0].makeBeamReg('bench/beam.reg', freq='mid', to_null=True)
# beamReg = 'bench/beam.reg'
#################################################################
# Get online model
if sourcedb == '':
if not os.path.exists('bench/tgts_bench.skydb'):
fwhm = MSs.getListObj()[0].getFWHM(freq='min')
radeg = phasecentre[0]
decdeg = phasecentre[1]
# get model the size of the image (radius=fwhm/2)
os.system(
'wget -O bench/tgts_bench.skymodel "https://lcs165.lofar.eu/cgi-bin/gsmv1.cgi?coord=%f,%f&radius=%f&unit=deg"'
% (radeg, decdeg, fwhm / 2.)) # ASTRON
lsm = lsmtool.load(
'bench/tgts_bench.skymodel') #, beamMS=MSs.getListObj()[0])
lsm.remove('I<1')
lsm.write('bench/tgts_bench.skymodel', clobber=True)
os.system(
'makesourcedb outtype="blob" format="<" in=bench/tgts_bench.skymodel out=bench/tgts_bench.skydb'
)
apparent = False
sourcedb = 'bench/tgts_bench.skydb'
#################################################################################################
# Add model to MODEL_DATA
# copy sourcedb into each MS to prevent concurrent access from multiprocessing to the sourcedb
sourcedb_basename = sourcedb.split('/')[-1]
for MS in MSs.getListStr():
lib_util.check_rm(MS + '/' + sourcedb_basename)
logger.debug('Copy: ' + sourcedb + ' -> ' + MS)
# make mask
im = lib_img.Image(imagename + '-MFS-image.fits')
im.makeMask(threshisl=3)
logger.info('Cleaning w/ mask...')
s.add('wsclean -continue -reorder -temp-dir /dev/shm -name ' + imagename + ' -size ' + str(size) + ' ' + str(size) + ' -j '+str(s.max_processors)+' -baseline-averaging 3 \
-scale 5arcsec -weight briggs 0.0 -niter 100000 -no-update-model-required -minuv-l 30 -mgain 0.85 -clean-border 1 \
-auto-threshold 0.1 -fits-mask ' +im.maskname+' \
-join-channels -fit-spectral-pol 2 -channels-out 10 -save-source-list ' +MSs.getStrWsclean(), \
log='wscleanB.log', commandType='wsclean', processors = 'max')
s.run(check=True)
os.system('cat logs/wscleanB.log | grep "background noise"')
# make new mask
im.makeMask(threshisl=5)
# apply mask
import lsmtool
logger.info('Predict (apply mask)...')
lsm = lsmtool.load(imagename + '-sources.txt')
lsm.select('%s == True' % (imagename + '-mask.fits'))
cRA, cDEC = MSs.getListObj()[0].getPhaseCentre()
lsm.select(lsm.getDistance(cRA, cDEC) > 0.1) # remove very centra part
lsm.group('every')
lsm.write(imagename + '-sources-cut.txt',
format='makesourcedb',
clobber=True)
del lsm
logger.info("Done.")
def main(input_image, input_skymodel_pb, input_bright_skymodel_pb, output_root,
vertices_file, threshisl=5.0, threshpix=7.5, rmsbox=(150, 50),
rmsbox_bright=(35, 7), adaptive_rmsbox=True,
use_adaptive_threshold=False, adaptive_thresh=75.0, beamMS=None,
peel_bright=False):
"""
Filter the input sky model so that they lie in islands in the image
Parameters
----------
input_image : str
Filename of input image to use to detect sources for filtering. Ideally, this
should be a flat-noise image (i.e., without primary-beam correction)
input_skymodel_pb : str
Filename of input makesourcedb sky model, with primary-beam correction
input_bright_skymodel_pb : str
Filename of input makesourcedb sky model of bright sources only, with primary-
beam correction
output_root : str
Root of filename of output makesourcedb sky models. Output filenames will be
output_root+'.apparent_sky.txt' and output_root+'.true_sky.txt'
vertices_file : str
Filename of file with vertices
threshisl : float, optional
Value of thresh_isl PyBDSF parameter
threshpix : float, optional
Value of thresh_pix PyBDSF parameter
rmsbox : tuple of floats, optional
Value of rms_box PyBDSF parameter
rmsbox_bright : tuple of floats, optional
Value of rms_box_bright PyBDSF parameter
adaptive_rmsbox : tuple of floats, optional
Value of adaptive_rms_box PyBDSF parameter
use_adaptive_threshold : bool, optional
If True, use an adaptive threshold estimated from the negative values in
the image
adaptive_thresh : float, optional
If adaptive_rmsbox is True, this value sets the threshold above
which a source will use the small rms box
peel_bright : bool, optional
If True, bright sources were peeled, so add then back before filtering
"""
if rmsbox is not None and isinstance(rmsbox, str):
rmsbox = eval(rmsbox)
if isinstance(rmsbox_bright, str):
rmsbox_bright = eval(rmsbox_bright)
adaptive_rmsbox = misc.string2bool(adaptive_rmsbox)
use_adaptive_threshold = misc.string2bool(use_adaptive_threshold)
if isinstance(beamMS, str):
beamMS = misc.string2list(beamMS)
peel_bright = misc.string2bool(peel_bright)
# Try to set the TMPDIR evn var to a short path, to ensure we do not hit the length
# limits for socket paths (used by the mulitprocessing module). We try a number of
# standard paths (the same ones used in the tempfile Python library)
old_tmpdir = os.environ["TMPDIR"]
for tmpdir in ['/tmp', '/var/tmp', '/usr/tmp']:
if os.path.exists(tmpdir):
os.environ["TMPDIR"] = tmpdir
break
# Run PyBDSF to make a mask for grouping
if use_adaptive_threshold:
# Get an estimate of the rms by running PyBDSF to make an rms map
img = bdsf.process_image(input_image, mean_map='zero', rms_box=rmsbox,
thresh_pix=threshpix, thresh_isl=threshisl,
thresh='hard', adaptive_rms_box=adaptive_rmsbox,
adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright,
rms_map=True, quiet=True, stop_at='isl')
# Find min and max pixels
max_neg_val = abs(np.min(img.ch0_arr))
max_neg_pos = np.where(img.ch0_arr == np.min(img.ch0_arr))
max_pos_val = abs(np.max(img.ch0_arr))
max_pos_pos = np.where(img.ch0_arr == np.max(img.ch0_arr))
# Estimate new thresh_isl from min pixel value's sigma, but don't let
# it get higher than 1/2 of the peak's sigma
threshisl_neg = 2.0 * max_neg_val / img.rms_arr[max_neg_pos][0]
max_sigma = max_pos_val / img.rms_arr[max_pos_pos][0]
if threshisl_neg > max_sigma / 2.0:
threshisl_neg = max_sigma / 2.0
# Use the new threshold only if it is larger than the user-specified one
if threshisl_neg > threshisl:
threshisl = threshisl_neg
img = bdsf.process_image(input_image, mean_map='zero', rms_box=rmsbox,
thresh_pix=threshpix, thresh_isl=threshisl,
thresh='hard', adaptive_rms_box=adaptive_rmsbox,
adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright,
atrous_do=True, atrous_jmax=3, rms_map=True, quiet=True)
emptysky = False
if img.nisl > 0:
maskfile = input_image + '.mask'
img.export_image(outfile=maskfile, clobber=True, img_type='island_mask')
# Construct polygon needed to trim the mask to the sector
header = pyfits.getheader(maskfile, 0)
w = wcs.WCS(header)
RAind = w.axis_type_names.index('RA')
Decind = w.axis_type_names.index('DEC')
vertices = misc.read_vertices(vertices_file)
RAverts = vertices[0]
Decverts = vertices[1]
verts = []
for RAvert, Decvert in zip(RAverts, Decverts):
ra_dec = np.array([[0.0, 0.0, 0.0, 0.0]])
ra_dec[0][RAind] = RAvert
ra_dec[0][Decind] = Decvert
verts.append((w.wcs_world2pix(ra_dec, 0)[0][RAind], w.wcs_world2pix(ra_dec, 0)[0][Decind]))
hdu = pyfits.open(maskfile, memmap=False)
data = hdu[0].data
# Rasterize the poly
data_rasertize = data[0, 0, :, :]
data_rasertize = misc.rasterize(verts, data_rasertize)
data[0, 0, :, :] = data_rasertize
hdu[0].data = data
hdu.writeto(maskfile, overwrite=True)
# Now filter the sky model using the mask made above
if len(beamMS) > 1:
# Select the best MS for the beam attenuation
ms_times = []
for ms in beamMS:
tab = pt.table(ms, ack=False)
ms_times.append(np.mean(tab.getcol('TIME')))
tab.close()
ms_times_sorted = sorted(ms_times)
mid_time = ms_times_sorted[int(len(ms_times)/2)]
beam_ind = ms_times.index(mid_time)
else:
beam_ind = 0
try:
s = lsmtool.load(input_skymodel_pb, beamMS=beamMS[beam_ind])
except astropy.io.ascii.InconsistentTableError:
emptysky = True
if peel_bright:
try:
# If bright sources were peeled before imaging, add them back
s_bright = lsmtool.load(input_bright_skymodel_pb, beamMS=beamMS[beam_ind])
# Rename the bright sources, removing the '_sector_*' added previously
# (otherwise the '_sector_*' text will be added every iteration,
# eventually making for very long source names)
new_names = [name.split('_sector')[0] for name in s_bright.getColValues('Name')]
s_bright.setColValues('Name', new_names)
if not emptysky:
s.concatenate(s_bright)
else:
s = s_bright
emptysky = False
except astropy.io.ascii.InconsistentTableError:
pass
if not emptysky:
s.select('{} == True'.format(maskfile)) # keep only those in PyBDSF masked regions
if len(s) == 0:
emptysky = True
else:
# Write out apparent and true-sky models
del(img) # helps reduce memory usage
s.group(maskfile) # group the sky model by mask islands
s.write(output_root+'.true_sky.txt', clobber=True)
s.write(output_root+'.apparent_sky.txt', clobber=True, applyBeam=True)
else:
emptysky = True
if emptysky:
# No sources cleaned/found in image, so just make a dummy sky model with single,
# very faint source at center
dummylines = ["Format = Name, Type, Patch, Ra, Dec, I, SpectralIndex, LogarithmicSI, "
"ReferenceFrequency='100000000.0', MajorAxis, MinorAxis, Orientation\n"]
ra, dec = img.pix2sky((img.shape[-2]/2.0, img.shape[-1]/2.0))
if ra < 0.0:
ra += 360.0
ra = misc.ra2hhmmss(ra)
sra = str(ra[0]).zfill(2)+':'+str(ra[1]).zfill(2)+':'+str("%.6f" % (ra[2])).zfill(6)
dec = misc.dec2ddmmss(dec)
decsign = ('-' if dec[3] < 0 else '+')
sdec = decsign+str(dec[0]).zfill(2)+'.'+str(dec[1]).zfill(2)+'.'+str("%.6f" % (dec[2])).zfill(6)
dummylines.append(',,p1,{0},{1}\n'.format(sra, sdec))
dummylines.append('s0c0,POINT,p1,{0},{1},0.00000001,'
'[0.0,0.0],false,100000000.0,,,\n'.format(sra, sdec))
with open(output_root+'.apparent_sky.txt', 'w') as f:
f.writelines(dummylines)
with open(output_root+'.true_sky.txt', 'w') as f:
f.writelines(dummylines)
# Set the TMPDIR env var back to its original value
os.environ["TMPDIR"] = old_tmpdir
#! /usr/bin/env python
# Runs an example of each operation
import lsmtool
import os
s = lsmtool.load('tests/no_patches.sky')
def test_select():
print('Select individual sources with Stokes I fluxes above 1 Jy')
s.select('I > 1.0 Jy')
assert len(s) == 965
def test_transfer():
print('Transfer patches from patches.sky')
s.transfer('tests/patches.sky')
assert s.hasPatches
def test_remove():
print('Remove patches with total fluxes below 2 Jy')
s.remove('I < 2.0 Jy', aggregate='sum')
assert len(s) == 433
def test_upgroup():
print('Ungroup the skymodel')
s.ungroup()
assert ~s.hasPatches
def skymodel_files_are_similar(file1, file2, atol, rtol, verbosity):
"""
Compare two skymodel files. Skymodels are considered similar if their
statistics are similar. This function uses LSMTool to determine the skymodel
statistics. The skymodels are considered to be similar, if the relevant
statistics differ less than ``atol`` or ``rtol``.
:param str file1: Filename of the first skymodel
:param str file2: Filename of the second skymodel
:param float atol: Absolute tolerance threshold
:param float rtol: Relative tolerance threshold
:param int verbosity: Verbosity level, higher is more verbose
:return bool: True is skymodels file are similar, else False
"""
def compare_stats(stats, atol, rtol):
"""
Compare statistics generated by LSMTool. Check if the flux ratio is
close to 1, and that the position offsets are less than ``atol``
:param float atol: Absolute tolerance used to compare position offsets
:param float rtol: Relative tolerance used to compare flux ratio
:return bool: True if statistics are similar, else False
"""
if not np.isclose(stats["meanRatio"], 1, rtol=rtol):
return False
if not np.isclose(stats["meanRAOffsetDeg"], 0, atol=atol):
return False
if not np.isclose(stats["meanDecOffsetDeg"], 0, atol=atol):
return False
return True
logger.debug("Comparing skymodels '%s' and '%s'", file1, file2)
sm1 = lsmtool.load(file1)
sm2 = lsmtool.load(file2)
stats = sm1.compare(sm2)
if not stats:
agree = False
else:
# We're only interested in a few statistics
stats = {
key: stats[key]
for key in ("meanRatio", "meanRAOffsetDeg", "meanDecOffsetDeg")
}
agree = compare_stats(stats, atol, rtol)
if agree:
logger.info("Skymodel files '%s' and '%s' are similar", file1, file2)
else:
if stats:
if verbosity > 0:
logger.error(
"Skymodel files '%s' and '%s' differ (atol=%s, rtol=%s): %s",
file1,
file2,
atol,
rtol,
stats,
)
else:
logger.error(
"Skymodel files '%s' and '%s' differ (atol=%s, rtol=%s)",
file1,
file2,
atol,
rtol,
)
else:
logger.error("Failed to compare skymodel files '%s' and '%s'",
file1, file2)
return agree
def _set_up_directions(parset, bands, dry_run=False, test_run=False,
reset_directions=[], reset_operations=[]):
"""
Sets up directions (facets)
Parameters
----------
parset : dict
Parset containing processing parameters
bands : list of Band instances
Vis data
dry_run : bool, optional
If True, do not run pipelines. All parsets, etc. are made as normal
test_run : bool, optional
If True, use test settings. These settings are for testing purposes
only and will not produce useful results
reset_directions : list of str, optional
List of direction names to be reset
reset_operations : list of str, optional
Llist of operations to be reset
Returns
-------
directions : List of Direction instances
All directions to be used by the run() function
direction_groups : List of lists of Direction instances
Groups of directions to be selfcal-ed
"""
dir_parset = parset['direction_specific']
max_radius_deg = dir_parset['max_radius_deg']
ref_band = bands[-1]
if dir_parset['faceting_skymodel'] is not None:
import lsmtool
log.info("Using {} as sky model for source avoidance and DDE calibrator "
"selection (if desired)".format(dir_parset['faceting_skymodel']))
initial_skymodel = lsmtool.load(dir_parset['faceting_skymodel'])
else:
log.info("Building local sky model for source avoidance and DDE calibrator "
"selection (if desired)...")
initial_skymodel = factor.directions.make_initial_skymodel(ref_band)
log.info('Setting up directions...')
directions = _initialize_directions(parset, initial_skymodel, ref_band,
max_radius_deg=max_radius_deg, dry_run=dry_run)
# Check with user
if parset['interactive']:
print("\nFacet and DDE calibrator regions saved. Please check that they\n"
"are OK before continuing. You can edit the directions file and\n"
"continue; FACTOR will pick up any changes to it. Note: if you\n"
"choose not to continue and you let FACTOR generate the directions\n"
"internally, you must delete the FACTOR-made directions file\n"
"(dir_working/factor_directions.txt) before restarting if you want\n"
"to FACTOR to regenerate it\n")
prompt = "Continue processing (y/n)? "
answ = raw_input(prompt)
while answ.lower() not in ['y', 'n', 'yes', 'no']:
answ = raw_input(prompt)
if answ.lower() in ['n', 'no']:
log.info('Exiting...')
sys.exit(0)
else:
# Continue processing, but first re-initialize the directions to
# pick up any changes the user made to the directions file
directions = _initialize_directions(parset, initial_skymodel,
ref_band, max_radius_deg=max_radius_deg, dry_run=dry_run)
# Warn user if they've specified a direction to reset that does not exist
direction_names = [d.name for d in directions]
for name in reset_directions:
if name not in direction_names and name != 'field':
log.warn('Direction {} was specified for resetting but does not '
'exist in current list of directions'.format(name))
# Load previously completed operations (if any) and facetsubreset-specific
# attributes and save the state
for direction in directions:
direction.load_state()
direction.save_state()
# Select subset of directions to process
target_has_own_facet = dir_parset['target_has_own_facet']
if target_has_own_facet:
direction_names = [d.name for d in directions]
target = directions[direction_names.index('target')]
if dir_parset['ndir_process'] is not None:
if dir_parset['ndir_process'] < len(directions):
directions = directions[:dir_parset['ndir_process']]
# Make sure target is still included
direction_names = [d.name for d in directions]
if target_has_own_facet and 'target' not in direction_names:
directions.append(target)
# Warn user if reimaging is to be done but they are not processing
# the full field
if parset['imaging_specific']['reimage_selfcaled']:
log.warn("The reimage_selfcaled parameter is True but all directions "
"will not be processed. If you're interested in only a single "
"target in the last facet, then re-imaging will not improve results.")
# Set various direction attributes
for i, direction in enumerate(directions):
# Set direction sky model
direction.set_skymodel(initial_skymodel.copy())
# Set peeling flag (i.e., facet calibrator should be peeled before facet
# is imaged)
total_flux_jy, peak_flux_jy_bm = direction.get_cal_fluxes()
effective_flux_jy = peak_flux_jy_bm * (total_flux_jy / peak_flux_jy_bm)**0.667
if (effective_flux_jy > parset['calibration_specific']['peel_flux_jy'] or
direction.is_outlier):
direction.find_peel_skymodel()
if direction.peel_skymodel is not None:
if not direction.is_outlier:
direction.peel_calibrator = True
log.info('Direction {0} will be peeled using sky model: {1}'.format(
direction.name, direction.peel_skymodel))
else:
if direction.is_outlier:
log.error('Direction {} was specified as an outlier source '
'but an appropriate sky model is not available'.format(direction.name))
sys.exit(1)
else:
log.warning('The flux density of direction {} exceeds peel_flux_Jy '
'but an appropriate peeling sky model is not available. '
'This direction will go through normal self calibration '
'instead'.format(direction.name))
# Set full correlation solve
if effective_flux_jy > parset['calibration_specific']['solve_all_correlations_flux_jy']:
if not parset['calibration_specific']['spline_smooth2d']:
log.error('The option spline_smooth2d must be enabled to use '
'XY and YX correlations during the slow gain solve')
sys.exit(1)
direction.solve_all_correlations = True
# Set skip_facet_imaging flag
direction.skip_facet_imaging = parset['imaging_specific']['skip_facet_imaging']
# Set field center to that of first band (all bands have the same phase
# center)
direction.field_ra = bands[0].ra
direction.field_dec = bands[0].dec
# Reset state if specified
if direction.name in reset_directions:
direction.do_reset = True
if len(reset_operations) > 0:
direction.reset_operations = reset_operations
else:
direction.reset_operations = (direction.completed_operations[:] +
direction.started_operations[:])
else:
direction.do_reset = False
# Select directions to selfcal, excluding outliers and target
if target_has_own_facet:
# Make sure target is not a DDE calibrator and is at end of directions list
selfcal_directions = [d for d in directions if d.name != target.name and
not d.is_outlier and not d.peel_calibrator]
directions = [d for d in directions if d.name != target.name] + [target]
else:
selfcal_directions = [d for d in directions if not d.is_outlier and
not d.peel_calibrator]
if dir_parset['ndir_selfcal'] is not None:
if dir_parset['ndir_selfcal'] <= len(selfcal_directions):
selfcal_directions = selfcal_directions[:dir_parset['ndir_selfcal']]
# Divide directions into groups for selfcal
direction_groups = factor.directions.group_directions(selfcal_directions,
n_per_grouping=dir_parset['groupings'], allow_reordering=dir_parset['allow_reordering'])
return directions, direction_groups
def _set_up_directions(parset,
bands,
dry_run=False,
test_run=False,
reset_directions=[],
reset_operations=[]):
"""
Sets up directions (facets)
Parameters
----------
parset : dict
Parset containing processing parameters
bands : list of Band instances
Vis data
dry_run : bool, optional
If True, do not run pipelines. All parsets, etc. are made as normal
test_run : bool, optional
If True, use test settings. These settings are for testing purposes
only and will not produce useful results
reset_directions : list of str, optional
List of direction names to be reset
reset_operations : list of str, optional
Llist of operations to be reset
Returns
-------
directions : List of Direction instances
All directions to be used by the run() function
direction_groups : List of lists of Direction instances
Groups of directions to be selfcal-ed
"""
dir_parset = parset['direction_specific']
max_radius_deg = dir_parset['max_radius_deg']
ref_band = bands[-1]
if dir_parset['faceting_skymodel'] is not None:
import lsmtool
log.info(
"Using {} as sky model for source avoidance and DDE calibrator "
"selection (if desired)".format(dir_parset['faceting_skymodel']))
initial_skymodel = lsmtool.load(dir_parset['faceting_skymodel'])
else:
log.info(
"Building local sky model for source avoidance and DDE calibrator "
"selection (if desired)...")
initial_skymodel = factor.directions.make_initial_skymodel(ref_band)
log.info('Setting up directions...')
directions = _initialize_directions(parset,
initial_skymodel,
ref_band,
max_radius_deg=max_radius_deg,
dry_run=dry_run)
# Check with user
if parset['interactive']:
print(
"\nFacet and DDE calibrator regions saved. Please check that they\n"
"are OK before continuing. You can edit the directions file and\n"
"continue; FACTOR will pick up any changes to it. Note: if you\n"
"choose not to continue and you let FACTOR generate the directions\n"
"internally, you must delete the FACTOR-made directions file\n"
"(dir_working/factor_directions.txt) before restarting if you want\n"
"FACTOR to regenerate it\n")
prompt = "Continue processing (y/n)? "
answ = raw_input(prompt)
while answ.lower() not in ['y', 'n', 'yes', 'no']:
answ = raw_input(prompt)
if answ.lower() in ['n', 'no']:
log.info('Exiting...')
sys.exit(0)
else:
# Continue processing, but first re-initialize the directions to
# pick up any changes the user made to the directions file
directions = _initialize_directions(parset,
initial_skymodel,
ref_band,
max_radius_deg=max_radius_deg,
dry_run=dry_run)
# Warn user if they've specified a direction to reset that does not exist
direction_names = [d.name for d in directions]
for name in reset_directions:
if name not in direction_names and name != 'field':
log.warn('Direction {} was specified for resetting but does not '
'exist in current list of directions'.format(name))
# Load previously completed operations (if any) and facetsubreset-specific
# attributes and save the state
for direction in directions:
direction.load_state()
direction.save_state()
# Select subset of directions to process
target_has_own_facet = dir_parset['target_has_own_facet']
if target_has_own_facet:
direction_names = [d.name for d in directions]
target = directions[direction_names.index('target')]
if dir_parset['ndir_process'] is not None:
if dir_parset['ndir_process'] < len(directions):
directions = directions[:dir_parset['ndir_process']]
# Make sure target is still included
direction_names = [d.name for d in directions]
if target_has_own_facet and 'target' not in direction_names:
directions.append(target)
# Set various direction attributes
for i, direction in enumerate(directions):
# Set direction sky model
direction.set_skymodel(initial_skymodel.copy())
# Set peeling flag (i.e., facet calibrator should be peeled before facet
# is imaged)
total_flux_jy, peak_flux_jy_bm = direction.get_cal_fluxes()
effective_flux_jy = peak_flux_jy_bm * (total_flux_jy /
peak_flux_jy_bm)**0.667
if (effective_flux_jy > parset['calibration_specific']['peel_flux_jy']
or direction.is_outlier):
direction.find_peel_skymodel()
if direction.peel_skymodel is not None:
if not direction.is_outlier:
direction.peel_calibrator = True
log.info(
'Direction {0} will be peeled using sky model: {1}'.format(
direction.name, direction.peel_skymodel))
else:
if direction.is_outlier:
log.error(
'Direction {} was specified as an outlier source '
'but an appropriate sky model is not available'.format(
direction.name))
sys.exit(1)
else:
log.warning(
'The flux density of direction {} exceeds peel_flux_Jy '
'but an appropriate peeling sky model is not available. '
'This direction will go through normal self calibration '
'instead'.format(direction.name))
# Set full correlation solve
if effective_flux_jy > parset['calibration_specific'][
'solve_all_correlations_flux_jy']:
if not parset['calibration_specific']['spline_smooth2d']:
log.error('The option spline_smooth2d must be enabled to use '
'XY and YX correlations during the slow gain solve')
sys.exit(1)
direction.solve_all_correlations = True
# Set field center to that of first band (all bands have the same phase
# center)
direction.field_ra = bands[0].ra
direction.field_dec = bands[0].dec
# Set initial name of column that contains SUBTRACTED_DATA_ALL
if parset['use_compression']:
# Since we compressed the input data, SUBTRACTED_DATA_ALL is now
# the DATA column
direction.subtracted_data_colname = 'DATA'
direction.use_compression = True
else:
direction.subtracted_data_colname = 'SUBTRACTED_DATA_ALL'
direction.use_compression = False
# Set any flagging parameters
direction.flag_abstime = parset['flag_abstime']
direction.flag_baseline = parset['flag_baseline']
direction.flag_freqrange = parset['flag_freqrange']
direction.flag_expr = parset['flag_expr']
# Reset state if specified
if direction.name in reset_directions:
direction.do_reset = True
if len(reset_operations) > 0:
direction.reset_operations = reset_operations
else:
direction.reset_operations = (
direction.completed_operations[:] +
direction.started_operations[:])
else:
direction.do_reset = False
# Select directions to selfcal, excluding outliers and target
if target_has_own_facet:
# Make sure target is not a DDE calibrator and is at end of directions list
selfcal_directions = [
d for d in directions if d.name != target.name and not d.is_outlier
and not d.peel_calibrator
]
directions = [d
for d in directions if d.name != target.name] + [target]
else:
selfcal_directions = [
d for d in directions if not d.is_outlier and not d.peel_calibrator
]
if dir_parset['ndir_selfcal'] is not None:
if dir_parset['ndir_selfcal'] <= len(selfcal_directions):
selfcal_directions = selfcal_directions[:
dir_parset['ndir_selfcal']]
# Divide directions into groups for selfcal
direction_groups = factor.directions.group_directions(
selfcal_directions,
n_per_grouping=dir_parset['groupings'],
allow_reordering=dir_parset['allow_reordering'])
return directions, direction_groups