def print_HAcov(self, png=None):
"""
some info on the MSs
"""
telescope = self.mssListObj[0].getTelescope()
if telescope == 'LOFAR':
telescope_coords = EarthLocation(lat=52.90889*u.deg, lon=6.86889*u.deg, height=0*u.m)
elif telescope == 'GMRT':
telescope_coords = EarthLocation(lat=19.0948*u.deg, lon=74.0493*u.deg, height=0*u.m)
else:
raise('Unknown Telescope.')
has = []; elevs = []
for ms in self.mssListObj:
time = np.mean(ms.getTimeRange())
time = Time( time/86400, format='mjd')
time.delta_ut1_utc = 0. # no need to download precise table for leap seconds
logger.info('%s (%s): Hour angle: %.1f hrs - Elev: %.2f (Sun distance: %.0f)' % (ms.nameMS,time.iso,ms.ha.deg/15.,ms.elev.deg,ms.sun_dist.deg))
has.append(ms.ha.deg/15.)
elevs.append(ms.elev.deg)
if png is not None:
import matplotlib.pyplot as pl
pl.figure(figsize=(6,6))
ax1 = pl.gca()
ax1.plot(has, elevs, 'ko')
ax1.set_xlabel('HA [hrs]')
ax1.set_ylabel('elevs [deg]')
logger.debug('Save plot: %s' % png)
pl.savefig(png)
def blank_image_fits(filename, maskname, outfile = None, inverse = False, blankval = 0.):
"""
Set to "blankval" all the pixels inside the given region
if inverse=True, set to "blankval" pixels outside region.
filename: fits file
region: ds9 region
outfile: output name
inverse: reverse region mask
blankval: pixel value to set
"""
if (outfile == None):
outfile = filename
with pyfits.open(maskname) as fits:
mask = fits[0].data
if (inverse): mask = ~(mask.astype(bool))
with pyfits.open(filename) as fits:
data = fits[0].data
assert mask.shape == data.shape # mask and data should be same shape
sum_before = np.sum(data)
data[mask] = blankval
logger.debug("%s: Blanking (%s): sum of values: %f -> %f" % (filename, maskname, sum_before, np.sum(data)))
fits.writeto(outfile, overwrite=True)
def makeBeamReg(self, outfile, pb_cut=None, to_null=False, freq='mid'):
"""
Create a ds9 region of the beam
outfile : str
output file
pb_cut : float, optional
diameter of the beam
to_null : bool, optional
arrive to the first null, not the FWHM
freq: min,max,med
which frequency to use to estimate the beam size
"""
logger.debug('Making PB region: '+outfile)
ra, dec = self.getPhaseCentre()
if pb_cut is None:
radius = self.getFWHM(freq=freq)/2.
else:
radius = pb_cut/2.
if to_null: radius *= 2 # rough estimation
s = Shape('circle', None)
s.coord_format = 'fk5'
s.coord_list = [ ra, dec, radius ] # ra, dec, radius
s.coord_format = 'fk5'
s.attr = ([], {'width': '2', 'point': 'cross',
'font': '"helvetica 16 normal roman"'})
s.comment = 'color=red text="beam"'
regions = pyregion.ShapeList([s])
lib_util.check_rm(outfile)
regions.write(outfile)
def run_losoto(s, c, h5s, parsets, plots_dir=None) -> object:
"""
s : scheduler
c : cycle name, e.g. "final"
h5s : lists of H5parm files or string of 1 h5parm
parsets : lists of parsets to execute
"""
logger.info("Running LoSoTo...")
h5out = 'cal-' + c + '.h5'
if type(h5s) is str: h5s = [h5s]
# convert from killMS
for i, h5 in enumerate(h5s):
if h5[-3:] == 'npz':
newh5 = h5.replace('.npz', '.h5')
s.add('killMS2H5parm.py -V --nofulljones %s %s ' % (newh5, h5),
log='losoto-' + c + '.log',
commandType="python",
processors='max')
s.run(check=True)
h5s[i] = newh5
# concat/move
if len(h5s) > 1:
check_rm(h5out)
s.add('H5parm_collector.py -V -s sol000 -o ' + h5out + ' ' +
' '.join(h5s),
log='losoto-' + c + '.log',
commandType="python",
processors='max')
s.run(check=True)
else:
os.system('cp -r %s %s' % (h5s[0], h5out))
check_rm('plots')
os.makedirs('plots')
for parset in parsets:
logger.debug('-- executing ' + parset + '...')
s.add('losoto -V ' + h5out + ' ' + parset,
log='losoto-' + c + '.log',
logAppend=True,
commandType="python",
processors='max')
s.run(check=True)
if plots_dir is None:
check_rm('plots-' + c)
os.system('mv plots plots-' + c)
else:
if not os.path.exists(plots_dir): os.system('mkdir ' + plots_dir)
os.system('mv plots/* ' + plots_dir)
check_rm('plots')
def getNchan(self):
"""
Find number of channels
"""
with tables.table(self.pathMS + "/SPECTRAL_WINDOW", ack=False) as t:
nchan = t.getcol("NUM_CHAN")
assert (nchan[0] == nchan).all() # all SpWs have same channels?
logger.debug("%s: channel number: %i", self.pathMS, nchan[0])
return nchan[0]
def getTimeInt(self):
"""
Get time interval in seconds
"""
with tables.table(self.pathMS, ack=False) as t:
nTimes = len(set(t.getcol("TIME")))
t_init, t_end = self.getTimeRange()
deltaT = (t_end - t_init) / nTimes
logger.debug("%s: time interval (seconds): %f", self.pathMS, deltaT)
return deltaT
def getChanband(self):
"""
Find bandwidth of a channel in Hz
"""
with tables.table(self.pathMS + "/SPECTRAL_WINDOW", ack=False) as t:
chan_w = t.getcol("CHAN_WIDTH")[0]
assert all(x == chan_w[0] for x in chan_w) # all chans have same width
logger.debug("%s: channel width (MHz): %f", self.pathMS,
chan_w[0] / 1.e6)
return chan_w[0]
def add(self,
cmd='',
log='',
logAppend=True,
commandType='',
processors=None):
"""
Add a command to the scheduler list
cmd: the command to run
log: log file name that can be checked at the end
logAppend: if True append, otherwise replace
commandType: can be a list of known command types as "BBS", "DPPP", ...
processors: number of processors to use, can be "max" to automatically use max number of processors per node
"""
if (log != ''):
log = self.log_dir + '/' + log
if (logAppend):
cmd += " >> "
else:
cmd += " > "
cmd += log + " 2>&1"
# if running wsclean add the string
if commandType == 'wsclean':
logger.debug('Running wsclean: %s' % cmd)
elif commandType == 'DPPP':
logger.debug('Running DPPP: %s' % cmd)
if (processors != None and processors == 'max'):
processors = self.max_processors
if self.qsub:
# if number of processors not specified, try to find automatically
if (processors == None):
processors = 1 # default use single CPU
if ("calibrate-stand-alone" == cmd[:21]):
processors = 1
if ("DPPP" == cmd[:5]):
processors = 1
if ("wsclean" == cmd[:7]):
processors = self.max_processors
if ("awimager" == cmd[:8]):
processors = self.max_processors
if (processors > self.max_processors):
processors = self.max_processors
self.action_list.append([str(processors), '\'' + cmd + '\''])
else:
self.action_list.append(cmd)
if (log != ""):
self.log_list.append((log, commandType))
def __exit__(self, type, value, traceback):
"""
Catch "Skip" errors, if not skipped, write to file after exited without exceptions.
"""
if type is None:
with open(self.filename, "a") as f:
f.write(self.__step__ + '\n')
logger.debug('Done: {}'.format(self.__step__))
return # No exception
if issubclass(type, Skip):
logger.warning('SKIP: {}'.format(self.__step__))
return True # Suppress special SkipWithBlock exception
def move(self, pathMSNew, overwrite=False, keepOrig=False):
"""
Move (or rename) the MS to another locus in the file system.
"""
logger.debug('Move: '+self.pathMS+' -> '+pathMSNew)
if overwrite == True:
lib_util.check_rm(pathMSNew)
if not os.path.exists(pathMSNew):
if keepOrig:
shutil.copytree(self.pathMS, pathMSNew)
else:
shutil.move(self.pathMS, pathMSNew)
self.setPathVariables(pathMSNew)
def blank_image_reg(filename,
region,
outfile=None,
inverse=False,
blankval=0.,
op="AND"):
"""
Set to "blankval" all the pixels inside the given region
if inverse=True, set to "blankval" pixels outside region.
If a list of region is provided the operation is applied to each region one after the other
filename: fits file
region: ds9 region or list of regions
outfile: output name
inverse: reverse final *combined* mask
blankval: pixel value to set
op: how to combine multiple regions with AND or OR
"""
if outfile == None: outfile = filename
if not type(region) is list: region = [region]
# open fits
with pyfits.open(filename) as fits:
origshape = fits[0].data.shape
header, data = flatten(fits)
sum_before = np.sum(data)
if (op == 'AND'):
total_mask = np.ones(shape=data.shape).astype(bool)
if (op == 'OR'):
total_mask = np.zeros(shape=data.shape).astype(bool)
for this_region in region:
# extract mask
r = pyregion.open(this_region)
mask = r.get_mask(header=header, shape=data.shape)
if (op == 'AND'):
total_mask = total_mask & mask
if (op == 'OR'):
total_mask = total_mask | mask
if (inverse):
total_mask = ~total_mask
data[total_mask] = blankval
# save fits
fits[0].data = data.reshape(origshape)
fits.writeto(outfile, overwrite=True)
logger.debug("%s: Blanking (%s): sum of values: %f -> %f" %
(filename, region, sum_before, np.sum(data)))
def __init__(self, coords, fluxes, kernel_size=0.2, look_distance=0.3, grouping_distance=0.03):
"""
coords: x,y coordinates for source positions
fluxes: total flux for each source
kernel_size: attenuate attraction, it this the flux times a gaussian of the distance with this as sigma [deg]
look_distance: max distance to look for nearby sources [deg]
grouping_distance: [deg]
"""
self.coords = np.array(coords)
self.fluxes = fluxes
self.kernel_size = kernel_size # deg
self.look_distance = look_distance # deg
self.grouping_distance = grouping_distance # deg orig: 0.01
self.past_coords = [np.copy(self.coords)]
self.n_iterations = 100
self.clusters = []
logger.debug("Grouper: kernel_size=%.1f; look_distance=%.1f; grouping_distance=%.2f" % (kernel_size,look_distance,grouping_distance) )
def add(self,
cmd='',
log='',
logAppend=True,
commandType='',
processors=None):
"""
Add a command to the scheduler list
cmd: the command to run
log: log file name that can be checked at the end
logAppend: if True append, otherwise replace
commandType: can be a list of known command types as "BBS", "DPPP", ...
processors: number of processors to use, can be "max" to automatically use max number of processors per node
"""
if (log != ''):
log = self.log_dir + '/' + log
if (logAppend):
cmd += " >> "
else:
cmd += " > "
cmd += log + " 2>&1"
# if running wsclean add the string
if commandType == 'wsclean':
logger.debug('Running wsclean: %s' % cmd)
elif commandType == 'DPPP':
logger.debug('Running DPPP: %s' % cmd)
elif commandType == 'singularity':
cmd = 'SINGULARITY_TMPDIR=/dev/shm singularity exec -B /tmp,/dev/shm,/localwork,/localwork.ssd,/home /home/fdg/node31/opt/src/lofar_sksp_ddf.simg ' + cmd
logger.debug('Running singularity: %s' % cmd)
elif commandType == 'python':
logger.debug('Running python: %s' % cmd)
if (processors != None and processors == 'max'):
processors = self.max_processors
if self.qsub:
# if number of processors not specified, try to find automatically
if (processors == None):
processors = 1 # default use single CPU
if ("DPPP" == cmd[:4]):
processors = 1
if ("wsclean" == cmd[:7]):
processors = self.max_processors
if (processors > self.max_processors):
processors = self.max_processors
self.action_list.append([str(processors), '\'' + cmd + '\''])
else:
self.action_list.append(cmd)
if (log != ""):
self.log_list.append((log, commandType))
def run_losoto(s, c, h5s, parsets):
"""
s : scheduler
c : cycle name, e.g. "final"
h5s : lists of H5parm files or string of 1 h5parm
parsets : lists of parsets to execute
"""
logger.info("Running LoSoTo...")
h5 = 'cal-' + c + '.h5'
if type(h5s) is str: h5s = [h5s]
# concat/move
if len(h5s) > 1:
check_rm("cal-" + c + ".h5")
s.add('H5parm_collector.py -V -s sol000 -o ' + h5 + ' ' +
' '.join(h5s),
log='losoto-' + c + '.log',
commandType="python",
processors='max')
s.run(check=True)
else:
os.system('cp -r %s %s' % (h5s[0], h5))
check_rm('plots')
os.makedirs('plots')
for parset in parsets:
logger.debug('-- executing ' + parset + '...')
s.add('losoto -V ' + h5 + ' ' + parset,
log='losoto-' + c + '.log',
logAppend=True,
commandType="python",
processors='max')
s.run(check=True)
check_rm('plots-' + c)
os.system('mv plots plots-' + c)
def columnAddSimilar(pathMS, columnNameNew, columnNameSimilar, dataManagerInfoNameNew, overwrite = False, fillWithOnes = True, comment = "", verbose = False):
# more to lib_ms
"""
Add a column to a MS that is similar to a pre-existing column (in shape, but not in values).
pathMS: path of the MS
columnNameNew: name of the column to be added
columnNameSimilar: name of the column from which properties are copied (e.g. "DATA")
dataManagerInfoNameNew: string value for the data manager info (DMI) keyword "NAME" (should be unique in the MS)
overwrite: whether or not to overwrite column 'columnNameNew' if it already exists
fillWithOnes: whether or not to fill the newly-made column with ones
verbose: whether or not to produce abundant output
"""
t = tables.table(pathMS, readonly = False)
if (columnExists(t, columnNameNew) and not overwrite):
logger.warning("Attempt to add column '" + columnNameNew + "' aborted, as it already exists and 'overwrite = False' in columnAddSimilar(...).")
else: # Either the column does not exist yet, or it does but overwriting is allowed.
# Remove column if necessary.
if (columnExists(t, columnNameNew)):
logger.info("Removing column '" + columnNameNew + "'...")
t.removecols(columnNameNew)
# Add column.
columnDescription = t.getcoldesc(columnNameSimilar)
dataManagerInfo = t.getdminfo(columnNameSimilar)
if (verbose):
logger.debug("columnDescription:")
logger.debug(columnDescription)
logger.debug("dataManagerInfo:")
logger.debug(dataManagerInfo)
columnDescription["comment"] = ""
# What about adding something here like:
#columnDescription["dataManagerGroup"] = ...?
dataManagerInfo["NAME"] = dataManagerInfoNameNew
if (verbose):
logger.debug("columnDescription (updated):")
logger.debug(columnDescription)
logger.debug("dataManagerInfo (updated):")
logger.debug(dataManagerInfo)
logger.info("Adding column '" + columnNameNew + "'...")
t.addcols(tables.makecoldesc(columnNameNew, columnDescription), dataManagerInfo)
# Fill with ones if desired.
if (fillWithOnes):
logger.info("Filling column '" + columnNameNew + "' with ones...")
columnDataSimilar = t.getcol(columnNameSimilar)
t.putcol(columnNameNew, np.ones_like(columnDataSimilar))
# Close the table to avoid that it is locked for further use.
t.close()
def make_voronoi_reg(directions,
fitsfile,
outdir_reg='regions',
out_mask='facet.fits',
png=None):
"""
Take a list of coordinates and an image and voronoi tesselate the sky.
It saves ds9 regions + fits mask of the facets
directions : array of Direction objects
firsfile : mask fits file to tassellate (used for coordinates and as template for the out_mask)
outdir_reg : dir where to save regions
out_mask : output mask with different numbers in each facet
png : output png file that shows the tassellation
"""
def closest_node(node, nodes):
"""
Return closest values to node from nodes
"""
nodes = np.asarray(nodes)
dist_2 = np.sum((nodes - node)**2, axis=1)
return np.argmin(dist_2)
logger.debug("Image used for tasselation reference: " + fitsfile)
fits = pyfits.open(fitsfile)
hdr, data = lib_img.flatten(fits)
w = pywcs.WCS(hdr)
pixsize = np.abs(hdr['CDELT1'])
# Get facets central pixels
ras = np.array([d.position_cal[0] for d in directions])
decs = np.array([d.position_cal[1] for d in directions])
x_fs, y_fs = w.all_world2pix(ras, decs, 0, ra_dec_order=True)
# keep trak of numbers in the direction names to name correctly patches in the fits files
# in this way Isl_patch_12 will have "12" into the fits for that patch.
nums = [d.isl_num for d in directions]
x_c = data.shape[0] / 2.
y_c = data.shape[1] / 2.
# Check if dir is in img, otherwise drop
idx_for_facet = []
for i, direction in enumerate(directions):
x, y = w.all_world2pix(ras[i], decs[i], 0, ra_dec_order=True)
if x < 0 or x > data.shape[0] or y < 0 or y > data.shape[1]:
logger.info(
'Direction %s is outside the primary beam and will not have a facet (it will still be a calibrator).'
% direction.name)
else:
idx_for_facet.append(i)
# convert to pixel space (voronoi must be in eucledian space)
x1 = 0
y1 = 0
x2 = data.shape[1] # note that y is before x in fits.data
y2 = data.shape[0]
# do tasselization
vor = Voronoi(
np.array((x_fs[idx_for_facet], y_fs[idx_for_facet])).transpose())
box = np.array([[x1, y1], [x2, y2]])
impoly = voronoi_finite_polygons_2d_box(vor, box)
# create fits mask (each region one number)
x, y = np.meshgrid(np.arange(x2),
np.arange(y2)) # make a canvas with coordinates
x, y = x.flatten(), y.flatten()
pixels = np.vstack((x, y)).T
data_facet = np.zeros(shape=data.shape)
for num, poly in zip(nums, impoly):
p = Path(poly)
pixels_region = p.contains_points(pixels)
data_facet[pixels_region.reshape(y2, x2)] = num
# put all values in each island equal to the closest region
struct = generate_binary_structure(2, 2)
data = binary_dilation(data, structure=struct,
iterations=3).astype(data.dtype) # expand masks
blobs, number_of_blobs = label(data.astype(int).squeeze(),
structure=[[1, 1, 1], [1, 1, 1], [1, 1, 1]])
center_of_masses = center_of_mass(data, blobs,
list(range(number_of_blobs + 1)))
for blob in range(1, number_of_blobs + 1):
# get closer facet
facet_num = closest_node(
center_of_masses[blob],
np.array([y_fs[idx_for_facet], x_fs[idx_for_facet]]).T)
# put all pixel of that mask to that facet value
data_facet[blobs == blob] = nums[facet_num]
# save fits mask
pyfits.writeto(out_mask, data_facet, hdr, overwrite=True)
# save regions
if not os.path.isdir(outdir_reg): os.makedirs(outdir_reg)
all_s = []
for i, poly in enumerate(impoly):
ra, dec = w.all_pix2world(poly[:, 0], poly[:, 1], 0, ra_dec_order=True)
coords = np.array([ra, dec]).T.flatten()
s = Shape('Polygon', None)
s.coord_format = 'fk5'
s.coord_list = coords # ra, dec, radius
s.coord_format = 'fk5'
s.attr = ([], {
'width': '2',
'point': 'cross',
'font': '"helvetica 16 normal roman"'
})
s.comment = 'color=red'
all_s.append(s)
regions = pyregion.ShapeList([s])
regionfile = outdir_reg + '/' + directions[
idx_for_facet[i]].name + '.reg'
regions.write(regionfile)
# add names for all.reg
for d in directions:
s = Shape('circle', None)
s.coord_format = 'fk5'
s.coord_list = [d.position_cal[0], d.position_cal[1],
0.01] # ra, dec, radius
s.coord_format = 'fk5'
s.attr = ([], {
'width': '1',
'point': 'cross',
'font': '"helvetica 16 normal roman"'
})
s.comment = 'color=white text="%s"' % d.name
all_s.append(s)
regions = pyregion.ShapeList(all_s)
regionfile = outdir_reg + '/all.reg'
regions.write(regionfile)
logger.debug(
'There are %i regions within the PB and %i outside (no facet).' %
(len(idx_for_facet), len(directions) - len(idx_for_facet)))
# plot tesselization
if png is not None:
import matplotlib.pyplot as pl
pl.figure(figsize=(8, 8))
ax1 = pl.gca()
voronoi_plot_2d(vor,
ax1,
show_vertices=True,
line_colors='black',
line_width=2,
point_size=4)
for i, d in enumerate(directions):
ax1.text(x_fs[i], y_fs[i], d.name, fontsize=15)
ax1.plot([x1, x1, x2, x2, x1], [y1, y2, y2, y1, y1])
ax1.set_xlabel('RA (pixel)')
ax1.set_ylabel('Dec (pixel)')
ax1.set_xlim(x1, x2)
ax1.set_ylim(y1, y2)
logger.debug('Save plot: %s' % png)
pl.savefig(png)
