def main(input_image_list, template_image, output_image, skip=False):
"""
Make a mosaic image
Parameters
----------
input_image_list : list
List of filenames of input FITS images to mosaic
template_image : str
Filename of mosaic template FITS image
output_image : str
Filename of output FITS image
skip : bool
If True, just copy input image and skip all other processing
"""
input_image_list = misc.string2list(input_image_list)
skip = misc.string2bool(skip)
if skip:
if os.path.exists(output_image):
os.remove(output_image)
shutil.copyfile(input_image_list[0], output_image)
return
# Load template and sector images and add them to mosaic
regrid_hdr = pyfits.open(template_image)[0].header
isum = pyfits.open(template_image)[0].data
xslices = [slice(0, int(isum.shape[0] / 2.0)),
slice(int(isum.shape[0] / 2.0), isum.shape[0])]
yslices = [slice(0, int(isum.shape[1] / 2.0)),
slice(int(isum.shape[1] / 2.0), isum.shape[1])]
for xs in xslices:
for ys in yslices:
wsum = np.zeros_like(isum[xs, ys])
for sector_image in input_image_list:
r = pyfits.open(sector_image)[0].section[xs, ys]
w = np.ones_like(r)
w[np.isnan(r)] = 0
r[np.isnan(r)] = 0
isum[xs, ys] += r
wsum += w
isum[xs, ys] /= wsum
del wsum, r, w
isum[np.isnan(isum)] = np.nan
hdu = pyfits.PrimaryHDU(header=regrid_hdr, data=isum)
hdu.writeto(output_image, overwrite=True)
def main(input_image, input_skymodel_pb, input_bright_skymodel_pb, output_root,
vertices_file, threshisl=5.0, threshpix=7.5, rmsbox=(150, 50),
rmsbox_bright=(35, 7), adaptive_rmsbox=True,
use_adaptive_threshold=False, adaptive_thresh=75.0, beamMS=None,
peel_bright=False):
"""
Filter the input sky model so that they lie in islands in the image
Parameters
----------
input_image : str
Filename of input image to use to detect sources for filtering. Ideally, this
should be a flat-noise image (i.e., without primary-beam correction)
input_skymodel_pb : str
Filename of input makesourcedb sky model, with primary-beam correction
input_bright_skymodel_pb : str
Filename of input makesourcedb sky model of bright sources only, with primary-
beam correction
output_root : str
Root of filename of output makesourcedb sky models. Output filenames will be
output_root+'.apparent_sky.txt' and output_root+'.true_sky.txt'
vertices_file : str
Filename of file with vertices
threshisl : float, optional
Value of thresh_isl PyBDSF parameter
threshpix : float, optional
Value of thresh_pix PyBDSF parameter
rmsbox : tuple of floats, optional
Value of rms_box PyBDSF parameter
rmsbox_bright : tuple of floats, optional
Value of rms_box_bright PyBDSF parameter
adaptive_rmsbox : tuple of floats, optional
Value of adaptive_rms_box PyBDSF parameter
use_adaptive_threshold : bool, optional
If True, use an adaptive threshold estimated from the negative values in
the image
adaptive_thresh : float, optional
If adaptive_rmsbox is True, this value sets the threshold above
which a source will use the small rms box
peel_bright : bool, optional
If True, bright sources were peeled, so add then back before filtering
"""
if rmsbox is not None and isinstance(rmsbox, str):
rmsbox = eval(rmsbox)
if isinstance(rmsbox_bright, str):
rmsbox_bright = eval(rmsbox_bright)
adaptive_rmsbox = misc.string2bool(adaptive_rmsbox)
use_adaptive_threshold = misc.string2bool(use_adaptive_threshold)
if isinstance(beamMS, str):
beamMS = misc.string2list(beamMS)
peel_bright = misc.string2bool(peel_bright)
# Try to set the TMPDIR evn var to a short path, to ensure we do not hit the length
# limits for socket paths (used by the mulitprocessing module). We try a number of
# standard paths (the same ones used in the tempfile Python library)
old_tmpdir = os.environ["TMPDIR"]
for tmpdir in ['/tmp', '/var/tmp', '/usr/tmp']:
if os.path.exists(tmpdir):
os.environ["TMPDIR"] = tmpdir
break
# Run PyBDSF to make a mask for grouping
if use_adaptive_threshold:
# Get an estimate of the rms by running PyBDSF to make an rms map
img = bdsf.process_image(input_image, mean_map='zero', rms_box=rmsbox,
thresh_pix=threshpix, thresh_isl=threshisl,
thresh='hard', adaptive_rms_box=adaptive_rmsbox,
adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright,
rms_map=True, quiet=True, stop_at='isl')
# Find min and max pixels
max_neg_val = abs(np.min(img.ch0_arr))
max_neg_pos = np.where(img.ch0_arr == np.min(img.ch0_arr))
max_pos_val = abs(np.max(img.ch0_arr))
max_pos_pos = np.where(img.ch0_arr == np.max(img.ch0_arr))
# Estimate new thresh_isl from min pixel value's sigma, but don't let
# it get higher than 1/2 of the peak's sigma
threshisl_neg = 2.0 * max_neg_val / img.rms_arr[max_neg_pos][0]
max_sigma = max_pos_val / img.rms_arr[max_pos_pos][0]
if threshisl_neg > max_sigma / 2.0:
threshisl_neg = max_sigma / 2.0
# Use the new threshold only if it is larger than the user-specified one
if threshisl_neg > threshisl:
threshisl = threshisl_neg
img = bdsf.process_image(input_image, mean_map='zero', rms_box=rmsbox,
thresh_pix=threshpix, thresh_isl=threshisl,
thresh='hard', adaptive_rms_box=adaptive_rmsbox,
adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright,
atrous_do=True, atrous_jmax=3, rms_map=True, quiet=True)
emptysky = False
if img.nisl > 0:
maskfile = input_image + '.mask'
img.export_image(outfile=maskfile, clobber=True, img_type='island_mask')
# Construct polygon needed to trim the mask to the sector
header = pyfits.getheader(maskfile, 0)
w = wcs.WCS(header)
RAind = w.axis_type_names.index('RA')
Decind = w.axis_type_names.index('DEC')
vertices = misc.read_vertices(vertices_file)
RAverts = vertices[0]
Decverts = vertices[1]
verts = []
for RAvert, Decvert in zip(RAverts, Decverts):
ra_dec = np.array([[0.0, 0.0, 0.0, 0.0]])
ra_dec[0][RAind] = RAvert
ra_dec[0][Decind] = Decvert
verts.append((w.wcs_world2pix(ra_dec, 0)[0][RAind], w.wcs_world2pix(ra_dec, 0)[0][Decind]))
hdu = pyfits.open(maskfile, memmap=False)
data = hdu[0].data
# Rasterize the poly
data_rasertize = data[0, 0, :, :]
data_rasertize = misc.rasterize(verts, data_rasertize)
data[0, 0, :, :] = data_rasertize
hdu[0].data = data
hdu.writeto(maskfile, overwrite=True)
# Now filter the sky model using the mask made above
if len(beamMS) > 1:
# Select the best MS for the beam attenuation
ms_times = []
for ms in beamMS:
tab = pt.table(ms, ack=False)
ms_times.append(np.mean(tab.getcol('TIME')))
tab.close()
ms_times_sorted = sorted(ms_times)
mid_time = ms_times_sorted[int(len(ms_times)/2)]
beam_ind = ms_times.index(mid_time)
else:
beam_ind = 0
try:
s = lsmtool.load(input_skymodel_pb, beamMS=beamMS[beam_ind])
except astropy.io.ascii.InconsistentTableError:
emptysky = True
if peel_bright:
try:
# If bright sources were peeled before imaging, add them back
s_bright = lsmtool.load(input_bright_skymodel_pb, beamMS=beamMS[beam_ind])
# Rename the bright sources, removing the '_sector_*' added previously
# (otherwise the '_sector_*' text will be added every iteration,
# eventually making for very long source names)
new_names = [name.split('_sector')[0] for name in s_bright.getColValues('Name')]
s_bright.setColValues('Name', new_names)
if not emptysky:
s.concatenate(s_bright)
else:
s = s_bright
emptysky = False
except astropy.io.ascii.InconsistentTableError:
pass
if not emptysky:
s.select('{} == True'.format(maskfile)) # keep only those in PyBDSF masked regions
if len(s) == 0:
emptysky = True
else:
# Write out apparent and true-sky models
del(img) # helps reduce memory usage
s.group(maskfile) # group the sky model by mask islands
s.write(output_root+'.true_sky.txt', clobber=True)
s.write(output_root+'.apparent_sky.txt', clobber=True, applyBeam=True)
else:
emptysky = True
if emptysky:
# No sources cleaned/found in image, so just make a dummy sky model with single,
# very faint source at center
dummylines = ["Format = Name, Type, Patch, Ra, Dec, I, SpectralIndex, LogarithmicSI, "
"ReferenceFrequency='100000000.0', MajorAxis, MinorAxis, Orientation\n"]
ra, dec = img.pix2sky((img.shape[-2]/2.0, img.shape[-1]/2.0))
if ra < 0.0:
ra += 360.0
ra = misc.ra2hhmmss(ra)
sra = str(ra[0]).zfill(2)+':'+str(ra[1]).zfill(2)+':'+str("%.6f" % (ra[2])).zfill(6)
dec = misc.dec2ddmmss(dec)
decsign = ('-' if dec[3] < 0 else '+')
sdec = decsign+str(dec[0]).zfill(2)+'.'+str(dec[1]).zfill(2)+'.'+str("%.6f" % (dec[2])).zfill(6)
dummylines.append(',,p1,{0},{1}\n'.format(sra, sdec))
dummylines.append('s0c0,POINT,p1,{0},{1},0.00000001,'
'[0.0,0.0],false,100000000.0,,,\n'.format(sra, sdec))
with open(output_root+'.apparent_sky.txt', 'w') as f:
f.writelines(dummylines)
with open(output_root+'.true_sky.txt', 'w') as f:
f.writelines(dummylines)
# Set the TMPDIR env var back to its original value
os.environ["TMPDIR"] = old_tmpdir
def main(h5parm1,
h5parm2,
outh5parm,
mode,
solset1='sol000',
solset2='sol000',
reweight=False,
cal_names=None,
cal_fluxes=None):
"""
Combines two h5parms
Parameters
----------
h5parm1 : str
Filename of h5parm 1
h5parm2 : str
Filename of h5parm 2
outh5parm : str
Filename of the output h5parm
mode : str
Mode to use when combining:
'p1a2' - phases from 1 and amplitudes from 2
'p1a1a2' - phases and amplitudes from 1 and amplitudes from 2 (amplitudes 1 and 2
are multiplied to create combined amplitudes)
'p1p2a2' - phases from 1 and phases and amplitudes from 2 (phases 2 are averaged
over XX and YY, then interpolated to time grid of 1 and summed)
solset1 : str, optional
Name of solset for h5parm1
solset2 : str, optional
Name of solset for h5parm2
reweight : bool, optional
If True, reweight the solutions by their detrended noise
cal_names : str or list, optional
List of calibrator names (for use in reweighting)
cal_fluxes : str or list, optional
List of calibrator flux densities (for use in reweighting)
"""
reweight = misc.string2bool(reweight)
cal_names = misc.string2list(cal_names)
cal_fluxes = misc.string2list(cal_fluxes)
# Make copies of the input h5parms (since they may be altered by steps below) and
# open them
with tempfile.TemporaryDirectory() as tmpdir:
h5parm1_copy = shutil.copy(h5parm1, tmpdir)
h5parm2_copy = shutil.copy(h5parm2, tmpdir)
h1 = h5parm(h5parm1_copy, readonly=False)
h2 = h5parm(h5parm2_copy, readonly=False)
ss1 = h1.getSolset(solset=solset1)
ss2 = h2.getSolset(solset=solset2)
# Initialize the output h5parm
if os.path.exists(outh5parm):
os.remove(outh5parm)
ho = h5parm(outh5parm, readonly=False)
sso = ho.makeSolset(solsetName='sol000', addTables=False)
if mode == 'p1a2':
# Take phases from 1 and amplitudes from 2
# Remove unneeded soltabs from 1 and 2, then copy
if 'amplitude000' in ss1.getSoltabNames():
st = ss1.getSoltab('amplitude000')
st.delete()
if 'phase000' in ss2.getSoltabNames():
st = ss2.getSoltab('phase000')
st.delete()
ss1.obj._f_copy_children(sso.obj, recursive=True, overwrite=True)
ss2.obj._f_copy_children(sso.obj, recursive=True, overwrite=True)
elif mode == 'p1a1a2':
# Take phases and amplitudes from 1 and amplitudes from 2 (amplitudes 1 and 2
# are multiplied to create combined values)
# First, copy phases and amplitudes from 1
ss1.obj._f_copy_children(sso.obj, recursive=True, overwrite=True)
# Then read amplitudes from 1 and 2, multiply them together, and store
st1 = ss1.getSoltab('amplitude000')
st2 = ss2.getSoltab('amplitude000')
sto = sso.getSoltab('amplitude000')
sto.setValues(st1.val * st2.val)
elif mode == 'p1p2a2':
# Take phases from 1 and phases and amplitudes from 2 (phases 2 are averaged
# over XX and YY, then interpolated to time grid of 1 and summed)
# First, copy phases from 1
ss1.obj._f_copy_children(sso.obj, recursive=True, overwrite=True)
# Read phases from 2, average XX and YY (using circmean), interpolate to match
# those from 1, and sum. Note: the interpolation is done in phase space (instead
# of real/imag space) since phase wraps are not expected to be present in the
# slow phases
st1 = ss1.getSoltab('phase000')
st2 = ss2.getSoltab('phase000')
axis_names = st1.getAxesNames()
time_ind = axis_names.index('time')
freq_ind = axis_names.index('freq')
axis_names = st2.getAxesNames()
pol_ind = axis_names.index('pol')
val2 = circmean(st2.val, axis=pol_ind) # average over XX and YY
if len(st2.time) > 1:
f = si.interp1d(st2.time,
val2,
axis=time_ind,
kind='nearest',
fill_value='extrapolate')
v1 = f(st1.time)
else:
v1 = val2
if len(st2.freq) > 1:
f = si.interp1d(st2.freq,
v1,
axis=freq_ind,
kind='linear',
fill_value='extrapolate')
vals = f(st1.freq) + st1.val
else:
vals = v1 + st1.val
sto = sso.getSoltab('phase000')
sto.setValues(vals)
# Copy amplitudes from 2
# Remove unneeded phase soltab from 2, then copy
if 'phase000' in ss2.getSoltabNames():
st = ss2.getSoltab('phase000')
st.delete()
ss2.obj._f_copy_children(sso.obj, recursive=True, overwrite=True)
else:
print('ERROR: mode not understood')
sys.exit(1)
# Close the files, copies are removed automatically
h1.close()
h2.close()
ho.close()
# Reweight
if reweight:
# Use the scatter on the solutions for weighting, with an additional scaling
# by the calibrator flux densities in each direction
ho = h5parm(outh5parm, readonly=False)
sso = ho.getSolset(solset='sol000')
# Reweight the phases. Reweighting doesn't work when there are too few samples,
# so check there are at least 10
soltab_ph = sso.getSoltab('phase000')
if len(soltab_ph.time) > 10:
# Set window size for std. dev. calculation. We try to get one of around
# 30 minutes, as that is roughly the timescale on which the global properties
# of the ionosphere are expected to change
delta_times = soltab_ph.time[1:] - soltab_ph.time[:-1]
timewidth = np.min(delta_times)
nstddev = min(251, max(11, int(1800 / timewidth)))
if nstddev % 2 == 0:
# Ensure window is odd
nstddev += 1
losoto.operations.reweight.run(soltab_ph,
mode='window',
nmedian=3,
nstddev=nstddev)
# Reweight the amplitudes
soltab_amp = sso.getSoltab('amplitude000')
if len(soltab_amp.time) > 10:
# Set window size for std. dev. calculation. We try to get one of around
# 90 minutes, as that is roughly the timescale on which the global properties
# of the beam errors are expected to change
delta_times = soltab_amp.time[1:] - soltab_amp.time[:-1]
timewidth = np.min(delta_times)
nstddev = min(251, max(11, int(5400 / timewidth)))
if nstddev % 2 == 0:
# Ensure window is odd
nstddev += 1
losoto.operations.reweight.run(soltab_amp,
mode='window',
nmedian=5,
nstddev=nstddev)
ho.close()
# Use the input calibrator flux densities to adjust the weighting done above
# to ensure that the average weights are proportional to the square of the
# calibrator flux densities
ho = h5parm(outh5parm, readonly=False)
sso = ho.getSolset(solset='sol000')
soltab_ph = sso.getSoltab('phase000')
soltab_amp = sso.getSoltab('amplitude000')
dir_names = [d.strip('[]') for d in soltab_ph.dir[:]]
cal_weights = []
for dir_name in dir_names:
cal_weights.append(cal_fluxes[cal_names.index(dir_name)])
cal_weights = [float(c) for c in cal_weights]
cal_weights = np.array(cal_weights)**2
# Convert weights to float64 from float16 to avoid clipping in the
# intermediate steps, and set flagged (weight = 0) solutions to NaN
# so they are not included in the calculations
weights_ph = np.array(soltab_ph.weight, dtype=np.float)
weights_amp = np.array(soltab_amp.weight, dtype=np.float)
weights_ph[weights_ph == 0.0] = np.nan
weights_amp[weights_amp == 0.0] = np.nan
# Reweight, keeping the median value of the weights the same (to avoid
# changing the overall normalization, which should be the inverse square of the
# uncertainty (scatter) in the solutions).
global_median_ph = np.nanmedian(weights_ph)
global_median_amp = np.nanmedian(weights_amp)
for d in range(len(dir_names)):
# Input data are [time, freq, ant, dir, pol] for slow amplitudes
# and [time, freq, ant, dir] for fast phases (scalarphase)
norm_factor = cal_weights[d] / np.nanmedian(weights_ph[:, :, :, d])
weights_ph[:, :, :, d] *= norm_factor
norm_factor = cal_weights[d] / np.nanmedian(weights_amp[:, :, :,
d, :])
weights_amp[:, :, :, d, :] *= norm_factor
weights_ph *= global_median_ph / np.nanmedian(weights_ph)
weights_amp *= global_median_amp / np.nanmedian(weights_amp)
weights_ph[np.isnan(weights_ph)] = 0.0
weights_amp[np.isnan(weights_amp)] = 0.0
# Clip to fit in float16 (required by LoSoTo)
float16max = 65504.0
weights_ph[weights_ph > float16max] = float16max
weights_amp[weights_amp > float16max] = float16max
# Write new weights
soltab_ph.setValues(weights_ph, weight=True)
soltab_amp.setValues(weights_amp, weight=True)
ho.close()
def main(msin,
msmod_list,
msin_column='DATA',
model_column='DATA',
out_column='DATA',
nr_outliers=0,
nr_bright=0,
use_compression=False,
peel_outliers=False,
peel_bright=False,
reweight=True,
starttime=None,
solint_sec=None,
solint_hz=None,
weights_colname="CAL_WEIGHT",
gainfile="",
uvcut_min=80.0,
uvcut_max=1e6,
phaseonly=True,
dirname=None,
quiet=True,
infix=''):
"""
Subtract sector model data
Parameters
----------
msin : str
Name of MS file from which subtraction will be done
msmod_list: list
List of model data MS filenames
msin_column : str, optional
Name of input column
model_column : str, optional
Name of model column
out_column : str, optional
Name of output column
nr_outliers : int, optional
Number of outlier sectors. Outlier sectors must be given after normal sectors
and bright-source sectors in msmod_list
nr_bright : int, optional
Number of bright-source sectors. Bright-source sectors must be given after normal
sectors but before outlier sectors in msmod_list
use_compression : bool, optional
If True, use Dysco compression
peel_outliers : bool, optional
If True, outliers are peeled before sector models are subtracted
peel_bright : bool, optional
If True, bright sources are peeled before sector models are subtracted
reweight : bool, optional
If True, reweight using the residuals
starttime : str, optional
Start time in JD seconds
infix : str, optional
Infix string used in filenames
"""
use_compression = misc.string2bool(use_compression)
peel_outliers = misc.string2bool(peel_outliers)
peel_bright = misc.string2bool(peel_bright)
nr_outliers = int(nr_outliers)
nr_bright = int(nr_bright)
solint_sec = float(solint_sec)
solint_hz = float(solint_hz)
uvcut_min = float(uvcut_min)
uvcut_max = float(uvcut_max)
uvcut = [uvcut_min, uvcut_max]
phaseonly = misc.string2bool(phaseonly)
reweight = misc.string2bool(reweight)
model_list = misc.string2list(msmod_list)
# Get the model data filenames, filtering any that do not have the right start time
if starttime is not None:
# Filter the list of models to include only ones for the given times
nrows_list = []
for msmod in model_list[:]:
tin = pt.table(msmod, readonly=True, ack=False)
starttime_chunk = np.min(tin.getcol('TIME'))
if not misc.approx_equal(
starttime_chunk, convert_mvt2mjd(starttime), tol=1.0):
# Remove files with start times that are not within 1 sec of the
# specified starttime
i = model_list.index(msmod)
model_list.pop(i)
else:
nrows_list.append(tin.nrows())
starttime_exact = convert_mjd2mvt(
starttime_chunk) # store exact value for use later
tin.close()
if len(set(nrows_list)) > 1:
print(
'subtract_sector_models: Model data files have differing number of rows...'
)
sys.exit(1)
nsectors = len(model_list)
if nsectors == 0:
print('subtract_sector_models: No model data found. Exiting...')
sys.exit(1)
print('subtract_sector_models: Found {} model data files'.format(nsectors))
# If starttime is given, figure out startrow and nrows for input MS file
tin = pt.table(msin, readonly=True, ack=False)
tarray = tin.getcol("TIME")
nbl = np.where(tarray == tarray[0])[0].size
tarray = None
if starttime is not None:
tapprox = convert_mvt2mjd(starttime_exact) - 100.0
approx_indx = np.where(tin.getcol('TIME') > tapprox)[0][0]
for tind, t in enumerate(tin.getcol('TIME')[approx_indx:]):
if convert_mjd2mvt(t) == starttime_exact:
startrow_in = tind + approx_indx
break
nrows_in = nrows_list[0]
else:
startrow_in = 0
nrows_in = tin.nrows()
tin.close()
# If outliers are to be peeled, do that first
if peel_outliers and nr_outliers > 0:
# Open input and output table
tin = pt.table(msin, readonly=True, ack=False)
root_filename = os.path.basename(msin)
msout = '{0}{1}_field'.format(root_filename, infix)
if infix != '':
# This implies we have a subrange of a full dataset, so use a model ms
# file as source for the copy (since otherwise we could copy the
# entire msin and not just the data for the correct subrange)
mssrc = model_list[-1]
else:
mssrc = msin
# Use subprocess to call 'cp' to ensure that the copied version has the
# default permissions (e.g., so it's not read only)
# TODO: Check for existence of `msout` could be removed. It should always
# be created in a different temporary directory by the CWL runner. If we
# don't trust the CWL runner, we might bail out if `msout` exists.
if os.path.exists(msout):
# File may exist from a previous iteration; delete it if so
misc.delete_directory(msout)
subprocess.check_call(
['cp', '-r', '-L', '--no-preserve=mode', mssrc, msout])
tout = pt.table(msout, readonly=False, ack=False)
# Define chunks based on available memory
fraction = float(nrows_in) / float(tin.nrows())
nchunks = get_nchunks(msin, nr_outliers, fraction, compressed=True)
nrows_per_chunk = int(nrows_in / nchunks)
startrows_tin = [startrow_in]
startrows_tmod = [0]
nrows = [nrows_per_chunk]
for i in range(1, nchunks):
if i == nchunks - 1:
nrow = nrows_in - (nchunks - 1) * nrows_per_chunk
else:
nrow = nrows_per_chunk
nrows.append(nrow)
startrows_tin.append(startrows_tin[i - 1] + nrows[i - 1])
startrows_tmod.append(startrows_tmod[i - 1] + nrows[i - 1])
print(
'subtract_sector_models: Using {} chunk(s) for peeling of outliers'
.format(nchunks))
for c, (startrow_tin, startrow_tmod,
nrow) in enumerate(zip(startrows_tin, startrows_tmod, nrows)):
# For each chunk, load data
datain = tin.getcol(msin_column, startrow=startrow_tin, nrow=nrow)
if use_compression:
# Replace flagged values with NaNs before compression
flags = tin.getcol('FLAG', startrow=startrow_tin, nrow=nrow)
flagged = np.where(flags)
datain[flagged] = np.NaN
datamod_list = []
for i, msmodel in enumerate(model_list[nsectors - nr_outliers:]):
tmod = pt.table(msmodel, readonly=True, ack=False)
datamod_list.append(
tmod.getcol(model_column,
startrow=startrow_tmod,
nrow=nrow))
tmod.close()
# Subtract sum of model data for this chunk
other_sectors_ind = list(range(nr_outliers))
datamod_all = None
for sector_ind in other_sectors_ind:
if datamod_all is None:
datamod_all = datamod_list[sector_ind].copy()
else:
datamod_all += datamod_list[sector_ind]
tout.putcol(out_column,
datain - datamod_all,
startrow=startrow_tmod,
nrow=nrow)
tout.flush()
tout.close()
tin.close()
# Now reset things for the imaging sectors
msin = msout
model_list = model_list[:-nr_outliers]
nsectors = len(model_list)
nr_outliers = 0
startrow_in = 0
datain = None
datamod_all = None
datamod_list = None
# Next, peel the bright sources if desired
if peel_bright and nr_bright > 0:
# Open input and output table
tin = pt.table(msin, readonly=True, ack=False)
root_filename = os.path.basename(msin)
msout = '{0}{1}_field_no_bright'.format(root_filename, infix)
if infix != '':
# This implies we have a subrange of a full dataset, so use a model ms
# file as source for the copy (since otherwise we could copy the
# entire msin and not just the data for the correct subrange)
mssrc = model_list[-1]
else:
mssrc = msin
# Use subprocess to call 'cp' to ensure that the copied version has the
# default permissions (e.g., so it's not read only)
# TODO: Check for existence of `msout` could be removed. It should always
# be created in a different temporary directory by the CWL runner. If we
# don't trust the CWL runner, we might bail out if `msout` exists.
if os.path.exists(msout):
# File may exist from a previous iteration; delete it if so
misc.delete_directory(msout)
subprocess.check_call(
['cp', '-r', '-L', '--no-preserve=mode', mssrc, msout])
tout = pt.table(msout, readonly=False, ack=False)
# Define chunks based on available memory
fraction = float(nrows_in) / float(tin.nrows())
nchunks = get_nchunks(msin, nr_bright, fraction, compressed=True)
nrows_per_chunk = int(nrows_in / nchunks)
startrows_tin = [startrow_in]
startrows_tmod = [0]
nrows = [nrows_per_chunk]
for i in range(1, nchunks):
if i == nchunks - 1:
nrow = nrows_in - (nchunks - 1) * nrows_per_chunk
else:
nrow = nrows_per_chunk
nrows.append(nrow)
startrows_tin.append(startrows_tin[i - 1] + nrows[i - 1])
startrows_tmod.append(startrows_tmod[i - 1] + nrows[i - 1])
print(
'subtract_sector_models: Using {} chunk(s) for peeling of bright '
'sources'.format(nchunks))
for c, (startrow_tin, startrow_tmod,
nrow) in enumerate(zip(startrows_tin, startrows_tmod, nrows)):
# For each chunk, load data
datain = tin.getcol(msin_column, startrow=startrow_tin, nrow=nrow)
if use_compression:
# Replace flagged values with NaNs before compression
flags = tin.getcol('FLAG', startrow=startrow_tin, nrow=nrow)
flagged = np.where(flags)
datain[flagged] = np.NaN
datamod_list = []
for i, msmodel in enumerate(model_list[nsectors - nr_bright:]):
tmod = pt.table(msmodel, readonly=True, ack=False)
datamod_list.append(
tmod.getcol(model_column,
startrow=startrow_tmod,
nrow=nrow))
tmod.close()
# Subtract sum of model data for this chunk
other_sectors_ind = list(range(nr_bright))
datamod_all = None
for sector_ind in other_sectors_ind:
if datamod_all is None:
datamod_all = datamod_list[sector_ind].copy()
else:
datamod_all += datamod_list[sector_ind]
tout.putcol(out_column,
datain - datamod_all,
startrow=startrow_tmod,
nrow=nrow)
tout.flush()
tout.close()
tin.close()
# Now reset things for the imaging sectors
msin = msout
model_list = model_list[:-nr_bright]
nsectors = len(model_list)
nr_bright = 0
startrow_in = 0
datain = None
datamod_all = None
datamod_list = None
# Open input table and define chunks based on available memory, making sure each
# chunk gives a full timeslot (needed for reweighting)
tin = pt.table(msin, readonly=True, ack=False)
fraction = float(nrows_in) / float(tin.nrows())
nchunks = get_nchunks(msin, nsectors, fraction, reweight=reweight)
nrows_per_chunk = int(nrows_in / nchunks)
while nrows_per_chunk % nbl > 0.0:
nrows_per_chunk -= 1
if nrows_per_chunk < nbl:
nrows_per_chunk = nbl
break
nchunks = int(np.ceil(nrows_in / nrows_per_chunk))
startrows_tin = [startrow_in]
startrows_tmod = [0]
nrows = [nrows_per_chunk]
for i in range(1, nchunks):
if i == nchunks - 1:
nrow = nrows_in - (nchunks - 1) * nrows_per_chunk
else:
nrow = nrows_per_chunk
nrows.append(nrow)
startrows_tin.append(startrows_tin[i - 1] + nrows[i - 1])
startrows_tmod.append(startrows_tmod[i - 1] + nrows[i - 1])
print(
'subtract_sector_models: Using {} chunk(s) for peeling of sector sources'
.format(nchunks))
# Open output tables
tout_list = []
for i, msmod in enumerate(model_list):
if nr_bright > 0 and nr_outliers > 0 and i == len(
model_list) - nr_outliers - nr_bright:
# Break so we don't open output tables for the outliers or bright sources
break
elif nr_outliers > 0 and i == len(model_list) - nr_outliers:
# Break so we don't open output tables for the outliers
break
elif nr_bright > 0 and i == len(model_list) - nr_bright:
# Break so we don't open output tables for the bright sources
break
msout = os.path.basename(msmod).rstrip('_modeldata')
if starttime is not None:
# Use a model ms file as source for the copy (since otherwise we could copy the
# entire msin and not just the data for the correct time range)
mssrc = model_list[-1]
else:
mssrc = msin
# Use subprocess to call 'cp' to ensure that the copied version has the
# default permissions (e.g., so it's not read only)
# TODO: Check for existence of `msout` could be removed. It should always
# be created in a different temporary directory by the CWL runner. If we
# don't trust the CWL runner, we might bail out if `msout` exists.
if os.path.exists(msout):
# File may exist from a previous iteration; delete it if so
misc.delete_directory(msout)
subprocess.check_call(
['cp', '-r', '-L', '--no-preserve=mode', mssrc, msout])
tout_list.append(pt.table(msout, readonly=False, ack=False))
# Process the data chunk by chunk
for c, (startrow_tin, startrow_tmod,
nrow) in enumerate(zip(startrows_tin, startrows_tmod, nrows)):
# For each chunk, load data
datain = tin.getcol(msin_column, startrow=startrow_tin, nrow=nrow)
flags = tin.getcol('FLAG', startrow=startrow_tin, nrow=nrow)
datamod_list = []
for i, msmodel in enumerate(model_list):
tmod = pt.table(msmodel, readonly=True, ack=False)
datamod_list.append(
tmod.getcol(model_column, startrow=startrow_tmod, nrow=nrow))
tmod.close()
# For each sector, subtract sum of model data of all other sectors
weights = None
for i, tout in enumerate(tout_list):
other_sectors_ind = list(range(nsectors))
other_sectors_ind.pop(i)
datamod_all = None
for sector_ind in other_sectors_ind:
if datamod_all is None:
datamod_all = datamod_list[sector_ind].copy()
else:
datamod_all += datamod_list[sector_ind]
if datamod_all is not None:
tout.putcol(out_column,
datain - datamod_all,
startrow=startrow_tmod,
nrow=nrow)
else:
tout.putcol(out_column,
datain,
startrow=startrow_tmod,
nrow=nrow)
if reweight:
# Also subtract sector's own model to make residual data for reweighting
if weights is None:
if datamod_all is None:
datamod_all = datamod_list[i]
else:
datamod_all += datamod_list[i]
covweights = CovWeights(model_list[0],
solint_sec,
solint_hz,
startrow_tmod,
nrow,
gainfile=gainfile,
uvcut=uvcut,
phaseonly=phaseonly,
dirname=dirname,
quiet=quiet)
coefficients = covweights.FindWeights(
datain - datamod_all, flags)
weights = covweights.calcWeights(coefficients)
covweights = None
tout.putcol(weights_colname,
weights,
startrow=startrow_tmod,
nrow=nrow)
tout.flush()
for tout in tout_list:
tout.close()
tin.close()
def main(input_image_list,
vertices_file_list,
output_image,
skip=False,
padding=1.1):
"""
Make a mosaic template image
Parameters
----------
input_image_list : list
List of filenames of input images to mosaic
vertices_file_list : list
List of filenames of input vertices files
output_image : str
Filename of output image
skip : bool
If True, skip all processing
padding : float
Fraction with which to increase the final mosaic size
"""
input_image_list = misc.string2list(input_image_list)
vertices_file_list = misc.string2list(vertices_file_list)
skip = misc.string2bool(skip)
if skip:
return
# Set up images used in mosaic
directions = []
for image_file, vertices_file in zip(input_image_list, vertices_file_list):
d = FITSImage(image_file)
d.vertices_file = vertices_file
d.blank()
directions.append(d)
# Prepare header for final gridding
mra = np.mean(np.array([d.get_wcs().wcs.crval[0] for d in directions]))
mdec = np.mean(np.array([d.get_wcs().wcs.crval[1] for d in directions]))
# Make a reference WCS and use it to find the extent in pixels
# needed for the combined image
rwcs = pywcs(naxis=2)
rwcs.wcs.ctype = directions[0].get_wcs().wcs.ctype
rwcs.wcs.cdelt = directions[0].get_wcs().wcs.cdelt
rwcs.wcs.crval = [mra, mdec]
rwcs.wcs.crpix = [1, 1]
xmin, xmax, ymin, ymax = 0, 0, 0, 0
for d in directions:
w = d.get_wcs()
ys, xs = np.where(d.img_data)
axmin, aymin, axmax, aymax = xs.min(), ys.min(), xs.max(), ys.max()
del xs, ys
for x, y in ((axmin, aymin), (axmax, aymin), (axmin, aymax), (axmax,
aymax)):
ra, dec = [float(f) for f in w.wcs_pix2world(x, y, 0)]
nx, ny = [float(f) for f in rwcs.wcs_world2pix(ra, dec, 0)]
xmin, xmax, ymin, ymax = min(nx, xmin), max(nx, xmax), min(
ny, ymin), max(ny, ymax)
xsize = int(xmax - xmin)
ysize = int(ymax - ymin)
xmax += int(xsize * (padding - 1.0) / 2.0)
xmin -= int(xsize * (padding - 1.0) / 2.0)
ymax += int(ysize * (padding - 1.0) / 2.0)
ymin -= int(ysize * (padding - 1.0) / 2.0)
xsize = int(xmax - xmin)
ysize = int(ymax - ymin)
rwcs.wcs.crpix = [-int(xmin) + 1, -int(ymin) + 1]
regrid_hdr = rwcs.to_header()
regrid_hdr['NAXIS'] = 2
regrid_hdr['NAXIS1'] = xsize
regrid_hdr['NAXIS2'] = ysize
for ch in ('BMAJ', 'BMIN', 'BPA', 'FREQ', 'RESTFREQ', 'EQUINOX'):
regrid_hdr[ch] = directions[0].img_hdr[ch]
regrid_hdr['ORIGIN'] = 'Raptor'
regrid_hdr['UNITS'] = 'Jy/beam'
regrid_hdr['TELESCOP'] = 'LOFAR'
isum = np.zeros([ysize, xsize])
hdu = pyfits.PrimaryHDU(header=regrid_hdr, data=isum)
hdu.writeto(output_image, overwrite=True)
def main(inh5parm, outh5parms, soltabname='phase000', insolset='sol000'):
"""
Combines two h5parms
Parameters
----------
inh5parm : str
Filename of h5parm to split
outh5parms : str
Filenames of split h5parms as comma-separated string
soltabname : str, optional
Name of soltab to use. If "gain" is in the name, phase and amplitudes are used
insolset : str, optional
Name of solset to split
"""
output_h5parm_list = misc.string2list(outh5parms)
nchunks = len(output_h5parm_list)
if nchunks == 1:
# If there is only one output file, just copy the input
shutil.copy(inh5parm, output_h5parm_list[0])
return
# Read input table
h5 = h5parm(inh5parm, readonly=True)
solset = h5.getSolset(insolset)
if 'gain' in soltabname:
soltab_amp = solset.getSoltab(soltabname.replace('gain', 'amplitude'))
soltab_ph = solset.getSoltab(soltabname.replace('gain', 'phase'))
else:
soltab_ph = solset.getSoltab(soltabname)
pointingNames = []
antennaNames = []
pointingDirections = []
antennaPositions = []
ants = solset.getAnt()
sous = solset.getSou()
for k, v in list(sous.items()):
if k not in pointingNames:
pointingNames.append(k)
pointingDirections.append(v)
for k, v in list(ants.items()):
if k not in antennaNames:
antennaNames.append(k)
antennaPositions.append(v)
# Read in times
times_fast = soltab_ph.time
if 'gain' in soltabname:
times_slow = soltab_amp.time
# Identify any gaps in time and put initial breaks there. We use median()
# instead of min() to find the solution interval (timewidth) because the
# division in time used during calibration to allow processing on multiple
# nodes can occasionally result in a few smaller solution intervals
delta_times = times_fast[1:] - times_fast[:-1] # time at center of solution interval
timewidth = np.median(delta_times)
gaps = np.where(delta_times > timewidth*1.2)
gaps_ind = gaps[0] + 1
gaps_ind = np.append(gaps_ind, np.array([len(times_fast)]))
# Add additional breaks to reach the desired number of chunks
if len(gaps_ind) >= nchunks:
gaps_ind = gaps_ind[:nchunks]
while len(gaps_ind) < nchunks:
# Find the largest existing gap
g_num_largest = 0
g_size_largest = 0
g_start = 0
for g_num, g_stop in enumerate(gaps_ind):
if g_stop - g_start > g_size_largest:
g_num_largest = g_num
g_size_largest = g_stop - g_start
g_start = g_stop
# Now split largest gap into two equal parts
if g_num_largest == 0:
g_start = 0
else:
g_start = gaps_ind[g_num_largest-1]
g_stop = gaps_ind[g_num_largest]
new_gap = g_start + int((g_stop - g_start) / 2)
gaps_ind = np.insert(gaps_ind, g_num_largest, np.array([new_gap]))
gaps_sec = []
for i, gind in enumerate(gaps_ind):
if i == nchunks-1:
gaps_sec.append(times_fast[-1])
else:
gaps_sec.append(times_fast[gind])
# Fill the output files
for i, outh5file in enumerate(output_h5parm_list):
if os.path.exists(outh5file):
os.remove(outh5file)
outh5 = h5parm(outh5file, readonly=False)
solsetOut = outh5.makeSolset('sol000')
# Store phases
if i == 0:
startval = times_fast[0]
else:
startval = gaps_sec[i-1]
if i == nchunks-1:
endval = times_fast[-1]
else:
endval = gaps_sec[i] - 0.5 # subtract 0.5 sec to ensure "[)" range
soltab_ph.setSelection(time={'min': startval, 'max': endval, 'step': 1})
solsetOut.makeSoltab('phase', 'phase000', axesNames=soltab_ph.getAxesNames(),
axesVals=[soltab_ph.getAxisValues(a) for a in soltab_ph.getAxesNames()],
vals=soltab_ph.getValues(retAxesVals=False),
weights=soltab_ph.getValues(weight=True, retAxesVals=False))
soltab_ph.clearSelection()
# Store amps
if 'gain' in soltabname:
if i == 0:
startval = times_slow[0]
else:
startval = gaps_sec[i-1]
if i == nchunks-1:
endval = times_slow[-1]
else:
endval = gaps_sec[i] - 0.5 # subtract 0.5 sec to ensure "[)" range
soltab_amp.setSelection(time={'min': startval, 'max': endval, 'step': 1})
solsetOut.makeSoltab('amplitude', 'amplitude000', axesNames=soltab_amp.getAxesNames(),
axesVals=[soltab_amp.getAxisValues(a) for a in soltab_amp.getAxesNames()],
vals=soltab_amp.getValues(retAxesVals=False),
weights=soltab_amp.getValues(weight=True, retAxesVals=False))
soltab_amp.clearSelection()
# Store metadata
sourceTable = solsetOut.obj._f_get_child('source')
antennaTable = solsetOut.obj._f_get_child('antenna')
antennaTable.append(list(zip(*(antennaNames, antennaPositions))))
sourceTable.append(list(zip(*(pointingNames, pointingDirections))))
outh5.close()