def test_mpccal_ICAL_onesource(self):
self.actualSetup(nsources=1, nvoronoi=1)
model = create_empty_image_like(self.theta_list[0].image)
if rsexecute.using_dask:
progress = None
else:
progress = self.progress
future_vis = rsexecute.scatter(self.all_skymodel_noniso_vis)
future_model = rsexecute.scatter(model)
future_theta_list = rsexecute.scatter(self.theta_list)
result = mpccal_skymodel_list_rsexecute_workflow(future_vis, future_model, future_theta_list,
mpccal_progress=progress,
nmajor=5,
context='2d',
algorithm='hogbom',
scales=[0, 3, 10],
fractional_threshold=0.15, threshold=0.05,
gain=0.1, niter=1000, psf_support=256,
deconvolve_facets=8, deconvolve_overlap=16,
deconvolve_taper='tukey')
(self.theta_list, residual) = rsexecute.compute(result, sync=True)
combined_model = calculate_skymodel_equivalent_image(self.theta_list)
psf_obs = invert_list_rsexecute_workflow([self.all_skymodel_noniso_vis], [model], context='2d', dopsf=True)
result = restore_list_rsexecute_workflow([combined_model], psf_obs, [(residual, 0.0)])
result = rsexecute.compute(result, sync=True)
if self.persist: export_image_to_fits(residual,
rascil_path('test_results/test_mpccal_ical_onesource_residual.fits'))
if self.persist: export_image_to_fits(result[0],
rascil_path('test_results/test_mpccal_ical_onesource_restored.fits'))
if self.persist: export_image_to_fits(combined_model,
rascil_path('test_results/test_mpccal_ical_onesource_deconvolved.fits'))
recovered_mpccal_components = find_skycomponents(result[0], fwhm=2, threshold=0.32, npixels=12)
def max_flux(elem):
return numpy.max(elem.flux)
recovered_mpccal_components = sorted(recovered_mpccal_components, key=max_flux, reverse=True)
assert recovered_mpccal_components[0].name == 'Segment 0', recovered_mpccal_components[0].name
assert numpy.abs(recovered_mpccal_components[0].flux[0, 0] - 1.138095494391862) < 1e-6, \
recovered_mpccal_components[0].flux[0, 0]
newscreen = create_empty_image_like(self.screen)
gaintables = [th.gaintable for th in self.theta_list]
newscreen, weights = grid_gaintable_to_screen(self.all_skymodel_noniso_blockvis, gaintables, newscreen)
if self.persist: export_image_to_fits(newscreen,
rascil_path('test_results/test_mpccal_ical_onesource_screen.fits'))
if self.persist: export_image_to_fits(weights,
rascil_path('test_results/test_mpccal_ical_onesource_screenweights.fits'))
rsexecute.close()
def progress(res, tl_list, gt_list, it, context='MPCCAL'):
print('Iteration %d' % it)
print(
qa_image(res,
context='%s residual image: iteration %d' %
(context, it)))
export_image_to_fits(
res,
rascil_path(
"test_results/low-sims-mpc-%s-residual_iteration%d_rmax%.1f.fits"
% (context, it, rmax)))
show_image(res,
title='%s residual image: iteration %d' % (context, it))
plt.show(block=block_plots)
combined_model = calculate_skymodel_equivalent_image(tl_list)
print(
qa_image(combined_model,
context='Combined model: iteration %d' % it))
export_image_to_fits(
combined_model,
rascil_path(
"test_results/low-sims-mpc-%s-model_iteration%d_rmax%.1f.fits"
% (context, it, rmax)))
plt.clf()
for i in range(len(tl_list)):
plt.plot(
numpy.angle(tl_list[i].gaintable.gain[:, :, 0, 0,
0]).flatten(),
numpy.angle(gt_list[i]['T'].gain[:, :, 0, 0, 0]).flatten(),
'.')
plt.xlabel('Current phase')
plt.ylabel('Update to phase')
plt.title("%s iteration%d: Change in phase" % (context, it))
plt.savefig(
rascil_path(
"test_results/low-sims-mpc-%s-skymodel-phase-change_iteration%d.jpg"
% (context, it)))
plt.show(block=block_plots)
return tl_list
def test_create_list(self):
if not self.casacore_available:
return
msfile = rascil_path("data/vis/xcasa.ms")
self.vis = create_blockvisibility_from_ms(msfile)
for v in self.vis:
assert v.vis.data.shape[-1] == 4
assert v.polarisation_frame.type == "circular"
def test_create_list_spectral(self):
if not self.casacore_available:
return
msfile = rascil_path("data/vis/ASKAP_example.ms")
vis_by_channel = list()
nchan_ave = 16
nchan = 192
for schan in range(0, nchan, nchan_ave):
max_chan = min(nchan, schan + nchan_ave)
v = create_visibility_from_ms(msfile, range(schan, max_chan))
vis_by_channel.append(v[0])
assert len(vis_by_channel) == 12
for v in vis_by_channel:
assert v.vis.data.shape[-1] == 4
assert v.polarisation_frame.type == "linear"
def test_create_list_slice_visibility(self):
if not self.casacore_available:
return
msfile = rascil_path("data/vis/ASKAP_example.ms")
vis_by_channel = list()
nchan_ave = 16
nchan = 192
for schan in range(0, nchan, nchan_ave):
max_chan = min(nchan, schan + nchan_ave)
v = create_visibility_from_ms(msfile, start_chan=schan, end_chan=max_chan - 1)
nchannels = len(numpy.unique(v[0].frequency))
assert nchannels == nchan_ave
vis_by_channel.append(v[0])
assert len(vis_by_channel) == 12
for v in vis_by_channel:
assert v.vis.data.shape[-1] == 4
assert v.polarisation_frame.type == "linear"
assert numpy.max(numpy.abs(v.vis)) > 0.0
assert numpy.max(numpy.abs(v.flagged_vis)) > 0.0
assert numpy.sum(v.weight) > 0.0
assert numpy.sum(v.flagged_weight) > 0.0
default='wstack',
help='Context: 2d|timeslice|wstack')
parser.add_argument('--nchan',
type=int,
default=40,
help='Number of channels to process')
parser.add_argument('--scheduler',
type=str,
default=None,
help='Dask scheduler')
args = parser.parse_args()
print(args)
# Put the results in the test_results directory
results_dir = rascil_path('test_results')
dask_dir = rascil_path('test_results/dask-work-space')
# Since the processing is distributed over multiple processes we have to tell each Dask worker
# where to send the log messages
def init_logging():
logging.basicConfig(
filename='%s/dprepb-pipeline.log' % results_dir,
filemode='a',
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
log = logging.getLogger()
logging.info("Starting Imaging pipeline")
phasecentre=phasecentre,
frequency=frequency,
polarisation_frame=PolarisationFrame('stokesI'),
radius=0.15)
all_components = apply_beam_to_skycomponent(original_gleam_components,
beam)
all_components = filter_skycomponents_by_flux(all_components,
flux_min=flux_limit)
all_components = sorted(all_components,
key=lambda comp: numpy.max(comp.flux),
reverse=True)
print("Number of components in simulation %d" % len(all_components))
screen = import_image_from_fits(
rascil_path('data/models/test_mpc_screen.fits'))
all_gaintables = create_gaintable_from_screen(block_vis, all_components,
screen)
all_skymodel = [
SkyModel(components=[all_components[i]], gaintable=all_gaintables[i])
for i, sm in enumerate(all_components)
]
#######################################################################################################
# Calculate visibility by using the predict_skymodel function which applies a different gaintable table
# for each skymodel. We do the calculation in chunks of nworkers skymodels.
all_skymodel_blockvis = copy_visibility(block_vis, zero=True)
all_skymodel_vis = convert_blockvisibility_to_visibility(
all_skymodel_blockvis)
"""
Simulation of observation for subsequent processing
"""
from rascil.data_models import rascil_path
results_dir = rascil_path('test_results')
dask_dir = rascil_path('test_results/dask-work-space')
import numpy
import logging
from astropy.coordinates import SkyCoord
from astropy import units as u
from rascil.data_models import SkyModel, PolarisationFrame, export_skymodel_to_hdf5, \
export_blockvisibility_to_hdf5
from rascil.processing_components.image.operations import create_empty_image_like
from rascil.processing_components import create_low_test_image_from_gleam, advise_wide_field, \
convert_blockvisibility_to_visibility, convert_visibility_to_blockvisibility
from rascil.workflows import predict_list_rsexecute_workflow, simulate_list_rsexecute_workflow, \
corrupt_list_rsexecute_workflow
from rascil.workflows.rsexecute.execution_support.rsexecute import rsexecute
def init_logging():
logging.basicConfig(
filename='%s/ska-pipeline.log' % results_dir,
filemode='a',
from rascil.processing_components import create_blockvisibility_from_ms, export_image_to_fits, qa_image,\
deconvolve_cube, restore_cube, create_image_from_visibility, convert_blockvisibility_to_visibility,\
convert_visibility_to_stokes
from rascil.workflows import invert_list_serial_workflow
import logging
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
log.addHandler(logging.StreamHandler(sys.stdout))
log.addHandler(logging.StreamHandler(sys.stderr))
if __name__ == '__main__':
results_dir = rascil_path('test_results')
bvt = create_blockvisibility_from_ms(rascil_path('data/vis/sim-2.ms'), start_chan=35, end_chan=39)[0]
bvt.configuration.diameter[...] = 35.0
vt = convert_blockvisibility_to_visibility(bvt)
vt = convert_visibility_to_stokes(vt)
a2r = numpy.pi / (180.0 * 3600.0)
model = create_image_from_visibility(vt, cellsize=20.0 * a2r, npixel=512,
polarisation_frame=PolarisationFrame('stokesIQUV'))
dirty, sumwt = invert_list_serial_workflow([vt], [model], context='2d')[0]
psf, sumwt = invert_list_serial_workflow([vt], [model], context='2d', dopsf=True)[0]
export_image_to_fits(dirty, '%s/rascil_imaging_sim_2_dirty.fits' % (results_dir))
export_image_to_fits(psf, '%s/rascil_imaging_sim_2_psf.fits' % (results_dir))
from rascil.processing_components import create_blockvisibility_from_ms, export_image_to_fits, qa_image,\
deconvolve_cube, restore_cube, create_image_from_visibility, convert_blockvisibility_to_visibility,\
convert_visibility_to_stokes
from rascil.workflows import invert_list_serial_workflow
import logging
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
log.addHandler(logging.StreamHandler(sys.stdout))
log.addHandler(logging.StreamHandler(sys.stderr))
if __name__ == '__main__':
results_dir = rascil_path('test_results')
bvt = create_blockvisibility_from_ms(rascil_path('data/vis/sim-2.ms'),
start_chan=35,
end_chan=39)[0]
bvt.configuration.diameter[...] = 35.0
vt = convert_blockvisibility_to_visibility(bvt)
vt = convert_visibility_to_stokes(vt)
a2r = numpy.pi / (180.0 * 3600.0)
model = create_image_from_visibility(
vt,
cellsize=20.0 * a2r,
npixel=512,
polarisation_frame=PolarisationFrame('stokesIQUV'))
def actualSetup(self, nsources=None, nvoronoi=None):
n_workers = 8
# Set up the observation: 10 minutes at transit, with 10s integration.
# Skip 5/6 points to avoid outstation redundancy
nfreqwin = 1
ntimes = 3
self.rmax = 2500.0
dec = -40.0 * u.deg
frequency = [1e8]
channel_bandwidth = [0.1e8]
times = numpy.linspace(-10.0, 10.0,
ntimes) * numpy.pi / (3600.0 * 12.0)
phasecentre = SkyCoord(ra=+0.0 * u.deg,
dec=dec,
frame='icrs',
equinox='J2000')
low = create_named_configuration('LOWBD2', rmax=self.rmax)
centre = numpy.mean(low.xyz, axis=0)
distance = numpy.hypot(low.xyz[:, 0] - centre[0],
low.xyz[:, 1] - centre[1],
low.xyz[:, 2] - centre[2])
lowouter = low.data[distance > 1000.0][::6]
lowcore = low.data[distance < 1000.0][::3]
low.data = numpy.hstack((lowcore, lowouter))
blockvis = create_blockvisibility(
low,
times,
frequency=frequency,
channel_bandwidth=channel_bandwidth,
weight=1.0,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
zerow=True)
vis = convert_blockvisibility_to_visibility(blockvis)
advice = advise_wide_field(vis, guard_band_image=2.0, delA=0.02)
cellsize = advice['cellsize']
npixel = advice['npixels2']
small_model = create_image_from_visibility(blockvis,
npixel=512,
frequency=frequency,
nchan=nfreqwin,
cellsize=cellsize,
phasecentre=phasecentre)
vis.data['imaging_weight'][...] = vis.data['weight'][...]
vis = weight_list_serial_workflow([vis], [small_model])[0]
vis = taper_list_serial_workflow([vis], 3 * cellsize)[0]
blockvis = convert_visibility_to_blockvisibility(vis)
# ### Generate the model from the GLEAM catalog, including application of the primary beam.
beam = create_image_from_visibility(blockvis,
npixel=npixel,
frequency=frequency,
nchan=nfreqwin,
cellsize=cellsize,
phasecentre=phasecentre)
beam = create_low_test_beam(beam, use_local=False)
flux_limit = 0.5
original_gleam_components = create_low_test_skycomponents_from_gleam(
flux_limit=flux_limit,
phasecentre=phasecentre,
frequency=frequency,
polarisation_frame=PolarisationFrame('stokesI'),
radius=0.15)
all_components = apply_beam_to_skycomponent(original_gleam_components,
beam)
all_components = filter_skycomponents_by_flux(all_components,
flux_min=flux_limit)
voronoi_components = filter_skycomponents_by_flux(all_components,
flux_min=1.5)
def max_flux(elem):
return numpy.max(elem.flux)
voronoi_components = sorted(voronoi_components,
key=max_flux,
reverse=True)
if nsources is not None:
all_components = [all_components[0]]
if nvoronoi is not None:
voronoi_components = [voronoi_components[0]]
self.screen = import_image_from_fits(
rascil_path('data/models/test_mpc_screen.fits'))
all_gaintables = create_gaintable_from_screen(blockvis, all_components,
self.screen)
gleam_skymodel_noniso = [
SkyModel(components=[all_components[i]],
gaintable=all_gaintables[i])
for i, sm in enumerate(all_components)
]
# ### Now predict the visibility for each skymodel and apply the gaintable for that skymodel,
# returning a list of visibilities, one for each skymodel. We then sum these to obtain
# the total predicted visibility. All images and skycomponents in the same skymodel
# get the same gaintable applied which means that in this case each skycomponent has a separate gaintable.
self.all_skymodel_noniso_vis = convert_blockvisibility_to_visibility(
blockvis)
ngroup = n_workers
future_vis = rsexecute.scatter(self.all_skymodel_noniso_vis)
chunks = [
gleam_skymodel_noniso[i:i + ngroup]
for i in range(0, len(gleam_skymodel_noniso), ngroup)
]
for chunk in chunks:
result = predict_skymodel_list_rsexecute_workflow(future_vis,
chunk,
context='2d',
docal=True)
work_vis = rsexecute.compute(result, sync=True)
for w in work_vis:
self.all_skymodel_noniso_vis.data['vis'] += w.data['vis']
assert numpy.max(
numpy.abs(self.all_skymodel_noniso_vis.data['vis'])) > 0.0
self.all_skymodel_noniso_blockvis = convert_visibility_to_blockvisibility(
self.all_skymodel_noniso_vis)
# ### Remove weaker of components that are too close (0.02 rad)
idx, voronoi_components = remove_neighbouring_components(
voronoi_components, 0.02)
model = create_image_from_visibility(blockvis,
npixel=npixel,
frequency=frequency,
nchan=nfreqwin,
cellsize=cellsize,
phasecentre=phasecentre)
# Use the gaintable for the brightest component as the starting gaintable
all_gaintables[0].gain[...] = numpy.conjugate(
all_gaintables[0].gain[...])
all_gaintables[0].gain[...] = 1.0 + 0.0j
self.theta_list = initialize_skymodel_voronoi(model,
voronoi_components,
all_gaintables[0])
from astropy.coordinates import SkyCoord
from astropy import units as u
from rascil.data_models import PolarisationFrame, rascil_path
from rascil.processing_components import create_visibility, export_image_to_fits, \
deconvolve_cube, restore_cube, create_named_configuration, create_test_image,\
create_image_from_visibility, advise_wide_field, invert_2d, predict_2d
import logging
log = logging.getLogger()
log.setLevel(logging.DEBUG)
log.addHandler(logging.StreamHandler(sys.stdout))
results_dir = rascil_path('test_results')
# Construct LOW core configuration
lowr3 = create_named_configuration('LOWBD2', rmax=750.0)
# We create the visibility. This just makes the uvw, time, antenna1, antenna2,
# weight columns in a table. We subsequently fill the visibility value in by
# a predict step.
times = numpy.zeros([1])
frequency = numpy.array([1e8])
channel_bandwidth = numpy.array([1e6])
phasecentre = SkyCoord(ra=+15.0 * u.deg,
dec=-45.0 * u.deg,
frame='icrs',
equinox='J2000')
return v
imaging_context = "ng"
vis_slices = 1
print('Using {}'.format(imaging_context))
predicted_vislist = predict_list_rsexecute_workflow(
vis_list,
dprepb_model,
context=imaging_context,
vis_slices=vis_slices,
verbosity=2)
corrupted_vislist = corrupt_list_rsexecute_workflow(predicted_vislist,
phase_error=1.0,
seed=180555)
corrupted_vislist = [
rsexecute.execute(print_max)(v) for v in corrupted_vislist
]
export_list = [
rsexecute.execute(export_blockvisibility_to_ms)(rascil_path(
'%s/ska-pipeline_simulation_vislist_%d.ms' % (results_dir, v)),
[corrupted_vislist[v]])
for v, _ in enumerate(corrupted_vislist)
]
print(
'About to run predict and corrupt to get corrupted visibility, and write files'
)
rsexecute.compute(export_list, sync=True)
rsexecute.close()
