def blank_image_fits(filename, maskname, outfile = None, inverse = False, blankval = 0.):
"""
Set to "blankval" all the pixels inside the given region
if inverse=True, set to "blankval" pixels outside region.
filename: fits file
region: ds9 region
outfile: output name
inverse: reverse region mask
blankval: pixel value to set
"""
if (outfile == None):
outfile = filename
with pyfits.open(maskname) as fits:
mask = fits[0].data
if (inverse): mask = ~(mask.astype(bool))
with pyfits.open(filename) as fits:
data = fits[0].data
assert mask.shape == data.shape # mask and data should be same shape
sum_before = np.sum(data)
data[mask] = blankval
logger.debug("%s: Blanking (%s): sum of values: %f -> %f" % (filename, maskname, sum_before, np.sum(data)))
fits.writeto(outfile, overwrite=True)
def getNchan(self):
"""
Find number of channels
"""
with tables.table(self.pathMS + "/SPECTRAL_WINDOW", ack=False) as t:
nchan = t.getcol("NUM_CHAN")
assert (nchan[0] == nchan).all() # all SpWs have same channels?
logger.debug("%s: channel number: %i", self.pathMS, nchan[0])
return nchan[0]
def getChanband(self):
"""
Find bandwidth of a channel in Hz
"""
with tables.table(self.pathMS + "/SPECTRAL_WINDOW", ack=False) as t:
chan_w = t.getcol("CHAN_WIDTH")[0]
assert all(x == chan_w[0] for x in chan_w) # all chans have same width
logger.debug("%s: channel width (MHz): %f", self.pathMS,
chan_w[0] / 1.e6)
return chan_w[0]
def getTimeInt(self):
"""
Get time interval in seconds
"""
with tables.table(self.pathMS, ack=False) as t:
nTimes = len(set(t.getcol("TIME")))
with tables.table(self.pathMS + "/OBSERVATION", ack=False) as t:
deltat = (t.getcol("TIME_RANGE")[0][1] -
t.getcol("TIME_RANGE")[0][0]) / nTimes
logger.debug("%s: time interval (seconds): %f", self.pathMS, deltat)
return deltat
def move(self, pathMSNew, overwrite=False, keepOrig=False):
"""
Move (or rename) the MS to another locus in the file system.
"""
logger.debug('Move: ' + self.pathMS + ' -> ' + pathMSNew)
if overwrite == True:
lib_util.check_rm(pathMSNew)
if not os.path.exists(pathMSNew):
if keepOrig:
shutil.copytree(self.pathMS, pathMSNew)
else:
shutil.move(self.pathMS, pathMSNew)
self.setPathVariables(pathMSNew)
def makeBeamReg(self, outfile, pb_cut=None, to_null=False):
"""
Create a ds9 region of the beam
outfile : str
output file
pb_cut : float, optional
diameter of the beam
to_null : bool, optional
arrive to the first null, not the FWHM
"""
logger.debug('Making PB region: ' + outfile)
ra, dec = self.getPhaseCentre()
if pb_cut is None:
if 'OUTER' in self.getObsMode():
size = 8. / 2.
elif 'SPARSE' in self.getObsMode():
size = 12. / 2.
elif 'INNER' in self.getObsMode():
size = 16. / 2.
else:
logger.error(
'Cannot find beam size, only LBA_OUTER or LBA_SPARSE_* are implemented. Assuming beam diameter = 8 deg.'
)
size = 8. / 2.
else:
size = pb_cut / 2.
if to_null: size *= 1.7 # rough estimation
s = Shape('circle', None)
s.coord_format = 'fk5'
s.coord_list = [ra, dec, size] # ra, dec, radius
s.coord_format = 'fk5'
s.attr = ([], {
'width': '2',
'point': 'cross',
'font': '"helvetica 16 normal roman"'
})
s.comment = 'color=red text="beam"'
regions = pyregion.ShapeList([s])
lib_util.check_rm(outfile)
regions.write(outfile)
def blank_image_reg(filename, region, outfile = None, inverse = False, blankval = 0., op = "AND"):
"""
Set to "blankval" all the pixels inside the given region
if inverse=True, set to "blankval" pixels outside region.
If a list of region is provided the operation is applied to each region one after the other
filename: fits file
region: ds9 region or list of regions
outfile: output name
inverse: reverse final *combined* mask
blankval: pixel value to set
op: how to combine multiple regions with AND or OR
"""
if outfile == None: outfile = filename
if not type(region) is list: region=[region]
# open fits
with pyfits.open(filename) as fits:
origshape = fits[0].data.shape
header, data = flatten(fits)
sum_before = np.sum(data)
if (op == 'AND'):
total_mask = np.ones(shape = data.shape).astype(bool)
if (op == 'OR'):
total_mask = np.zeros(shape = data.shape).astype(bool)
for this_region in region:
# extract mask
r = pyregion.open(this_region)
mask = r.get_mask(header=header, shape=data.shape)
if (op == 'AND'):
total_mask = total_mask & mask
if (op == 'OR'):
total_mask = total_mask | mask
if (inverse):
total_mask = ~total_mask
data[total_mask] = blankval
# save fits
fits[0].data = data.reshape(origshape)
fits.writeto(outfile, overwrite=True)
logger.debug("%s: Blanking (%s): sum of values: %f -> %f" % (filename, region, sum_before, np.sum(data)))
def run_losoto(s, c, h5s, parsets):
"""
s : scheduler
c : cycle name, e.g. "final"
h5s : lists of H5parm files
parsets : lists of parsets to execute
"""
logger.info("Running LoSoTo...")
# concat
if len(h5s) > 1:
h5 = 'cal-' + c + '.h5'
check_rm("cal-" + c + ".h5")
s.add('H5parm_collector.py -V -s sol000 -o ' + h5 + ' ' +
' '.join(h5s),
log='losoto-' + c + '.log',
commandType="python",
processors='max')
s.run(check=True)
else:
h5 = h5s[0]
check_rm('plots')
os.makedirs('plots')
for parset in parsets:
logger.debug('-- executing ' + parset + '...')
s.add('losoto -V ' + h5 + ' ' + parset,
log='losoto-' + c + '.log',
logAppend=True,
commandType="python",
processors='max')
s.run(check=True)
check_rm('plots-' + c)
os.system('mv plots plots-' + c)
def columnAddSimilar(pathMS,
columnNameNew,
columnNameSimilar,
dataManagerInfoNameNew,
overwrite=False,
fillWithOnes=True,
comment="",
verbose=False):
# more to lib_ms
"""
Add a column to a MS that is similar to a pre-existing column (in shape, but not in values).
pathMS: path of the MS
columnNameNew: name of the column to be added
columnNameSimilar: name of the column from which properties are copied (e.g. "DATA")
dataManagerInfoNameNew: string value for the data manager info (DMI) keyword "NAME" (should be unique in the MS)
overwrite: whether or not to overwrite column 'columnNameNew' if it already exists
fillWithOnes: whether or not to fill the newly-made column with ones
verbose: whether or not to produce abundant output
"""
t = tables.table(pathMS, readonly=False)
if (columnExists(t, columnNameNew) and not overwrite):
logger.warning(
"Attempt to add column '" + columnNameNew +
"' aborted, as it already exists and 'overwrite = False' in columnAddSimilar(...)."
)
else: # Either the column does not exist yet, or it does but overwriting is allowed.
# Remove column if necessary.
if (columnExists(t, columnNameNew)):
logger.info("Removing column '" + columnNameNew + "'...")
t.removecols(columnNameNew)
# Add column.
columnDescription = t.getcoldesc(columnNameSimilar)
dataManagerInfo = t.getdminfo(columnNameSimilar)
if (verbose):
logger.debug("columnDescription:")
logger.debug(columnDescription)
logger.debug("dataManagerInfo:")
logger.debug(dataManagerInfo)
columnDescription["comment"] = ""
# What about adding something here like:
#columnDescription["dataManagerGroup"] = ...?
dataManagerInfo["NAME"] = dataManagerInfoNameNew
if (verbose):
logger.debug("columnDescription (updated):")
logger.debug(columnDescription)
logger.debug("dataManagerInfo (updated):")
logger.debug(dataManagerInfo)
logger.info("Adding column '" + columnNameNew + "'...")
t.addcols(tables.makecoldesc(columnNameNew, columnDescription),
dataManagerInfo)
# Fill with ones if desired.
if (fillWithOnes):
logger.info("Filling column '" + columnNameNew + "' with ones...")
columnDataSimilar = t.getcol(columnNameSimilar)
t.putcol(columnNameNew, np.ones_like(columnDataSimilar))
# Close the table to avoid that it is locked for further use.
t.close()
def make_voronoi_reg(directions,
fitsfile,
outdir_reg='regions',
out_mask='facet.fits',
beam_reg=None,
png=None):
"""
Take a list of coordinates and an image and voronoi tesselate the sky.
It saves ds9 regions + fits mask of the facets
directions : dict with {'Dir_0':[ra,dec], 'Dir_1':[ra,dec]...} - note that the "Dir_##" naming is important
firsfile : mask fits file to tassellate (used for coordinates and to avoid splitting islands)
outdir* : dir where to save regions/masks
beam_reg : a ds9 region showing the the primary beam, exclude directions outside it
"""
def closest_node(node, nodes):
"""
Return closest values to node from nodes
"""
nodes = np.asarray(nodes)
dist_2 = np.sum((nodes - node)**2, axis=1)
return np.argmin(dist_2)
import lib_img
logger.debug("Image used for tasselation reference: " + fitsfile)
fits = pyfits.open(fitsfile)
hdr, data = lib_img.flatten(fits)
w = pywcs.WCS(hdr)
pixsize = np.abs(hdr['CDELT1'])
# Get facets central pixels
ras = np.array([directions[d][0].degree for d in directions])
decs = np.array([directions[d][1].degree for d in directions])
x_fs, y_fs = w.all_world2pix(ras, decs, 0, ra_dec_order=True)
# keep trak of numbers in the direction names to name correctly patches in the fits files
# in this way Dir_12 will have "12" into the fits for that patch.
nums = [int(d.split('_')[1]) for d in directions.keys()]
x_c = data.shape[0] / 2.
y_c = data.shape[1] / 2.
if beam_reg is None:
# no beam, use all directions for facets
idx_for_facet = range(len(directions))
else:
r = pyregion.open(beam_reg)
beam_mask = r.get_mask(header=hdr, shape=data.shape)
beamradius_pix = r[0].coord_list[2] / pixsize
idx_for_facet = []
for i, dd in enumerate(t):
if beam_mask[t['x'][i], t['y'][i]] == True:
idx_for_facet.append(i)
# convert to pixel space (voronoi must be in eucledian space)
x1 = 0
y1 = 0
x2 = data.shape[0]
y2 = data.shape[1]
# do tasselization
vor = Voronoi(
np.array((x_fs[idx_for_facet], y_fs[idx_for_facet])).transpose())
box = np.array([[x1, y1], [x2, y2]])
impoly = voronoi_finite_polygons_2d_box(vor, box)
# create fits mask (each region one number)
x, y = np.meshgrid(np.arange(x2),
np.arange(y2)) # make a canvas with coordinates
x, y = x.flatten(), y.flatten()
pixels = np.vstack((x, y)).T
data_facet = np.zeros(shape=data.shape)
for num, poly in zip(nums, impoly):
p = Path(poly)
pixels_region = p.contains_points(pixels)
# iterate through direction centres and find which one belongs to this region, then use the dir name to set the number
# this is important as the vornoi tassellation has to have the same names of the original tassellation
#for x,y,d in zip(x_fs, y_fs, directions.keys()):
# if pixels_region.reshape(x2,y2)[int(np.rint(x)),int(np.rint(y))] == True:
# num = int(d.split('_')[1])
# print num,x,y,d
data_facet[pixels_region.reshape(x2, y2)] = num
# put all values in each island equal to the closest region
struct = generate_binary_structure(2, 2)
data = binary_dilation(data, structure=struct,
iterations=3).astype(data.dtype) # expand masks
blobs, number_of_blobs = label(data.astype(int).squeeze(),
structure=[[1, 1, 1], [1, 1, 1], [1, 1, 1]])
center_of_masses = center_of_mass(data, blobs, range(number_of_blobs + 1))
for blob in xrange(1, number_of_blobs + 1):
# get closer facet
facet_num = closest_node(center_of_masses[blob],
np.array([y_fs, x_fs]).T)
# put all pixel of that mask to that facet value
data_facet[blobs == blob] = nums[facet_num]
# save fits mask
pyfits.writeto(out_mask, data_facet, hdr, overwrite=True)
# save regions
all_s = []
for i, poly in enumerate(impoly):
ra, dec = w.all_pix2world(poly[:, 0], poly[:, 1], 0, ra_dec_order=True)
coords = np.array([ra, dec]).T.flatten()
s = Shape('Polygon', None)
s.coord_format = 'fk5'
s.coord_list = coords # ra, dec, radius
s.coord_format = 'fk5'
s.attr = ([], {
'width': '2',
'point': 'cross',
'font': '"helvetica 16 normal roman"'
})
s.comment = 'color=red'
all_s.append(s)
regions = pyregion.ShapeList([s])
regionfile = outdir_reg + '/' + directions.keys()[
idx_for_facet[i]] + '.reg'
regions.write(regionfile)
# add names for all.reg
for d_name, d_coord in directions.iteritems():
s = Shape('circle', None)
s.coord_format = 'fk5'
s.coord_list = [d_coord[0].degree, d_coord[1].degree,
0.01] # ra, dec, radius
s.coord_format = 'fk5'
s.attr = ([], {
'width': '1',
'point': 'cross',
'font': '"helvetica 16 normal roman"'
})
s.comment = 'color=white text="%s"' % d_name
all_s.append(s)
regions = pyregion.ShapeList(all_s)
regionfile = outdir_reg + '/all.reg'
regions.write(regionfile)
logger.debug(
'There are %i regions within the PB and %i outside (no facet).' %
(len(idx_for_facet), len(directions) - len(idx_for_facet)))
# plot tesselization
if png is not None:
import matplotlib.pyplot as pl
pl.figure(figsize=(8, 8))
ax1 = pl.gca()
voronoi_plot_2d(vor,
ax1,
show_vertices=True,
line_colors='black',
line_width=2,
point_size=4)
for i, d in enumerate(directions):
ax1.text(x_fs[i], y_fs[i], d, fontsize=15)
if not beam_reg is None:
c1 = pl.Circle((x_c, y_c), beamradius_pix, color='g', fill=False)
ax1.add_artist(c1)
ax1.plot([x1, x1, x2, x2, x1], [y1, y2, y2, y1, y1])
ax1.set_xlabel('RA (pixel)')
ax1.set_ylabel('Dec (pixel)')
ax1.set_xlim(x1, x2)
ax1.set_ylim(y1, y2)
logger.debug('Save plot: %s' % png)
pl.savefig(png)