step = channel_average * channel_bandwidth
averaged_channel_bandwidth = step * numpy.ones_like(averaged_frequency)
print("Each averaged chunk has %d integrations of duration %.2f (s)" %
(nintegrations_per_chunk // time_average,
time_average * integration_time))
print("Each averaged chunk has %d channels of width %.3f (MHz)" %
(len(averaged_frequency), 1e-6 * averaged_channel_bandwidth[0]))
print("Processing %d time chunks in groups of %d" %
(len(start_times), args.ngroup_visibility))
cellsize = 1e-4 * (3000.0 / rmax)
model_graph = arlexecute.execute(create_image)(
cellsize=cellsize,
npixel=npixel,
frequency=averaged_frequency,
channel_bandwidth=averaged_channel_bandwidth,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
pole_model_graph = arlexecute.execute(create_image)(
cellsize=cellsize,
npixel=npixel,
frequency=averaged_frequency,
channel_bandwidth=averaged_channel_bandwidth,
phasecentre=pole,
polarisation_frame=PolarisationFrame("stokesI"))
# We process the chunks (and accumulate the images) in two stages to avoid a large final reduction
for chunk_start_time in chunk_start_times:
chunk_results = list()
chunk_pole_results = list()
def predict_list_arlexecute_workflow(vis_list,
model_imagelist,
context,
vis_slices=1,
facets=1,
gcfcf=None,
**kwargs):
"""Predict, iterating over both the scattered vis_list and image
The visibility and image are scattered, the visibility is predicted on each part, and then the
parts are assembled.
:param vis_list:
:param model_imagelist: Model used to determine image parameters
:param vis_slices: Number of vis slices (w stack or timeslice)
:param facets: Number of facets (per axis)
:param context: Type of processing e.g. 2d, wstack, timeslice or facets
:param gcfcg: tuple containing grid correction and convolution function
:param kwargs: Parameters for functions in components
:return: List of vis_lists
"""
if get_parameter(kwargs, "use_serial_predict", False):
from workflows.serial.imaging.imaging_serial import predict_list_serial_workflow
return [arlexecute.execute(predict_list_serial_workflow, nout=1) \
(vis_list=[vis_list[i]],
model_imagelist=[model_imagelist[i]], vis_slices=vis_slices,
facets=facets, context=context, gcfcf=gcfcf, **kwargs)[0]
for i, _ in enumerate(vis_list)]
assert len(vis_list) == len(
model_imagelist), "Model must be the same length as the vis_list"
# Predict_2d does not clear the vis so we have to do it here.
vis_list = zero_list_arlexecute_workflow(vis_list)
c = imaging_context(context)
vis_iter = c['vis_iterator']
predict = c['predict']
if facets % 2 == 0 or facets == 1:
actual_number_facets = facets
else:
actual_number_facets = facets - 1
def predict_ignore_none(vis, model, g):
if vis is not None:
assert isinstance(vis, Visibility), vis
assert isinstance(model, Image), model
return predict(vis, model, context=context, gcfcf=g, **kwargs)
else:
return None
if gcfcf is None:
gcfcf = [
arlexecute.execute(create_pswf_convolutionfunction)(m)
for m in model_imagelist
]
# Loop over all frequency windows
if facets == 1:
image_results_list = list()
for freqwin, vis_list in enumerate(vis_list):
if len(gcfcf) > 1:
g = gcfcf[freqwin]
else:
g = gcfcf[0]
# Create the graph to divide an image into facets. This is by reference.
# Create the graph to divide the visibility into slices. This is by copy.
sub_vis_lists = arlexecute.execute(visibility_scatter,
nout=vis_slices)(vis_list,
vis_iter,
vis_slices)
image_vis_lists = list()
# Loop over sub visibility
for sub_vis_list in sub_vis_lists:
# Predict visibility for this sub-visibility from this image
image_vis_list = arlexecute.execute(predict_ignore_none, pure=True, nout=1) \
(sub_vis_list, model_imagelist[freqwin], g)
# Sum all sub-visibilities
image_vis_lists.append(image_vis_list)
image_results_list.append(
arlexecute.execute(visibility_gather,
nout=1)(image_vis_lists, vis_list,
vis_iter))
return image_results_list
else:
image_results_list_list = list()
for freqwin, vis_list in enumerate(vis_list):
# Create the graph to divide an image into facets. This is by reference.
facet_lists = arlexecute.execute(image_scatter_facets,
nout=actual_number_facets**2)(
model_imagelist[freqwin],
facets=facets)
# Create the graph to divide the visibility into slices. This is by copy.
sub_vis_lists = arlexecute.execute(visibility_scatter,
nout=vis_slices)(vis_list,
vis_iter,
vis_slices)
facet_vis_lists = list()
# Loop over sub visibility
for sub_vis_list in sub_vis_lists:
facet_vis_results = list()
# Loop over facets
for facet_list in facet_lists:
# Predict visibility for this subvisibility from this facet
facet_vis_list = arlexecute.execute(predict_ignore_none,
pure=True,
nout=1)(sub_vis_list,
facet_list,
None)
facet_vis_results.append(facet_vis_list)
# Sum the current sub-visibility over all facets
facet_vis_lists.append(
arlexecute.execute(sum_predict_results)(facet_vis_results))
# Sum all sub-visibilities
image_results_list_list.append(
arlexecute.execute(visibility_gather,
nout=1)(facet_vis_lists, vis_list,
vis_iter))
return image_results_list_list
def weight_list_arlexecute_workflow(vis_list,
model_imagelist,
gcfcf=None,
weighting='uniform',
**kwargs):
""" Weight the visibility data
This is done collectively so the weights are summed over all vis_lists and then
corrected
:param vis_list:
:param model_imagelist: Model required to determine weighting parameters
:param weighting: Type of weighting
:param kwargs: Parameters for functions in graphs
:return: List of vis_graphs
"""
centre = len(model_imagelist) // 2
if gcfcf is None:
gcfcf = [
arlexecute.execute(create_pswf_convolutionfunction)(
model_imagelist[centre])
]
def grid_wt(vis, model, g):
if vis is not None:
if model is not None:
griddata = create_griddata_from_image(model)
griddata = grid_weight_to_griddata(vis, griddata, g[0][1])
return griddata
else:
return None
else:
return None
weight_list = [
arlexecute.execute(grid_wt, pure=True)(vis_list[i], model_imagelist[i],
gcfcf)
for i in range(len(vis_list))
]
merged_weight_grid = arlexecute.execute(griddata_merge_weights,
nout=len(vis_list))(weight_list)
def re_weight(vis, model, gd, g):
if gd is not None:
if vis is not None:
# Ensure that the griddata has the right axes so that the convolution
# function mapping works
agd = create_griddata_from_image(model)
agd.data = gd[0].data
vis = griddata_reweight(vis, agd, g[0][1])
return vis
else:
return None
else:
return vis
return [
arlexecute.execute(re_weight, nout=1)(v, model_imagelist[i],
merged_weight_grid, gcfcf)
for i, v in enumerate(vis_list)
]
def convolve_skymodel_list_arlexecute_workflow(obsvis,
skymodel_list,
context,
vis_slices=1,
facets=1,
gcfcf=None,
**kwargs):
"""Form residual image from observed visibility and a set of skymodel without calibration
This is similar to convolving the skymodel images with the PSF
:param vis_list: List of Visibility data models
:param skymodel_list: skymodel list
:param vis_slices: Number of vis slices (w stack or timeslice)
:param facets: Number of facets (per axis)
:param context: Type of processing e.g. 2d, wstack, timeslice or facets
:param gcfcg: tuple containing grid correction and convolution function
:param docal: Apply calibration table in skymodel
:param kwargs: Parameters for functions in components
:return: List of (image, weight) tuples)
"""
def ft_ift_sm(ov, sm, g):
assert isinstance(ov, Visibility), ov
assert isinstance(sm, SkyModel), sm
if g is not None:
assert len(g) == 2, g
assert isinstance(g[0], Image), g[0]
assert isinstance(g[1], ConvolutionFunction), g[1]
v = copy_visibility(ov)
v.data['vis'][...] = 0.0 + 0.0j
if len(sm.components) > 0:
if isinstance(sm.mask, Image):
comps = copy_skycomponent(sm.components)
comps = apply_beam_to_skycomponent(comps, sm.mask)
v = predict_skycomponent_visibility(v, comps)
else:
v = predict_skycomponent_visibility(v, sm.components)
if isinstance(sm.image, Image):
if numpy.max(numpy.abs(sm.image.data)) > 0.0:
if isinstance(sm.mask, Image):
model = copy_image(sm.image)
model.data *= sm.mask.data
else:
model = sm.image
v = predict_list_serial_workflow([v], [model],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=[g],
**kwargs)[0]
assert isinstance(sm.image, Image), sm.image
result = invert_list_serial_workflow([v], [sm.image],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=[g],
**kwargs)[0]
if isinstance(sm.mask, Image):
result[0].data *= sm.mask.data
return result
if gcfcf is None:
return [
arlexecute.execute(ft_ift_sm, nout=len(skymodel_list))(obsvis, sm,
None)
for ism, sm in enumerate(skymodel_list)
]
else:
return [
arlexecute.execute(ft_ift_sm, nout=len(skymodel_list))(obsvis, sm,
gcfcf[ism])
for ism, sm in enumerate(skymodel_list)
]
logging.info("Starting ical_list_arlexecute_workflow-pipeline")
arlexecute.set_client(get_dask_Client())
arlexecute.run(init_logging)
# Load data from previous simulation
vislist = import_blockvisibility_from_hdf5('gleam_simulation_vislist.hdf')
print(vislist[0])
cellsize = 0.001
npixel = 1024
pol_frame = PolarisationFrame("stokesI")
model_list = [
arlexecute.execute(create_image_from_visibility)(
v, npixel=1024, cellsize=cellsize, polarisation_frame=pol_frame)
for v in vislist
]
controls = create_calibration_controls()
controls['T']['first_selfcal'] = 1
controls['G']['first_selfcal'] = 3
controls['B']['first_selfcal'] = 4
controls['T']['timescale'] = 'auto'
controls['G']['timescale'] = 'auto'
controls['B']['timescale'] = 1e5
pp.pprint(controls)
def simulate_list_arlexecute_workflow(config='LOWBD2',
phasecentre=SkyCoord(ra=+15.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000'),
frequency=None, channel_bandwidth=None, times=None,
polarisation_frame=PolarisationFrame("stokesI"), order='frequency',
format='blockvis',
rmax=1000.0,
zerow=False):
""" A component to simulate an observation
The simulation step can generate a single BlockVisibility or a list of BlockVisibility's.
The parameter keyword determines the way that the list is constructed.
If order='frequency' then len(frequency) BlockVisibility's with all times are created.
If order='time' then len(times) BlockVisibility's with all frequencies are created.
If order = 'both' then len(times) * len(times) BlockVisibility's are created each with
a single time and frequency. If order = None then all data are created in one BlockVisibility.
The output format can be either 'blockvis' (for calibration) or 'vis' (for imaging)
:param config: Name of configuration: def LOWBDS-CORE
:param phasecentre: Phase centre def: SkyCoord(ra=+15.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000')
:param frequency: def [1e8]
:param channel_bandwidth: def [1e6]
:param times: Observing times in radians: def [0.0]
:param polarisation_frame: def PolarisationFrame("stokesI")
:param order: 'time' or 'frequency' or 'both' or None: def 'frequency'
:param format: 'blockvis' or 'vis': def 'blockvis'
:return: vis_list with different frequencies in different elements
"""
if format == 'vis':
create_vis = create_visibility
else:
create_vis = create_blockvisibility
if times is None:
times = [0.0]
if channel_bandwidth is None:
channel_bandwidth = [1e6]
if frequency is None:
frequency = [1e8]
conf = create_named_configuration(config, rmax=rmax)
if order == 'time':
log.debug("simulate_list_arlexecute_workflow: Simulating distribution in %s" % order)
vis_list = list()
for i, time in enumerate(times):
vis_list.append(arlexecute.execute(create_vis, nout=1)(conf, numpy.array([times[i]]),
frequency=frequency,
channel_bandwidth=channel_bandwidth,
weight=1.0, phasecentre=phasecentre,
polarisation_frame=polarisation_frame,
zerow=zerow))
elif order == 'frequency':
log.debug("simulate_list_arlexecute_workflow: Simulating distribution in %s" % order)
vis_list = list()
for j, _ in enumerate(frequency):
vis_list.append(arlexecute.execute(create_vis, nout=1)(conf, times,
frequency=numpy.array([frequency[j]]),
channel_bandwidth=numpy.array(
[channel_bandwidth[j]]),
weight=1.0, phasecentre=phasecentre,
polarisation_frame=polarisation_frame,
zerow=zerow))
elif order == 'both':
log.debug("simulate_list_arlexecute_workflow: Simulating distribution in time and frequency")
vis_list = list()
for i, _ in enumerate(times):
for j, _ in enumerate(frequency):
vis_list.append(arlexecute.execute(create_vis, nout=1)(conf, numpy.array([times[i]]),
frequency=numpy.array([frequency[j]]),
channel_bandwidth=numpy.array(
[channel_bandwidth[j]]),
weight=1.0, phasecentre=phasecentre,
polarisation_frame=polarisation_frame,
zerow=zerow))
elif order is None:
log.debug("simulate_list_arlexecute_workflow: Simulating into single %s" % format)
vis_list = list()
vis_list.append(arlexecute.execute(create_vis, nout=1)(conf, times, frequency=frequency,
channel_bandwidth=channel_bandwidth,
weight=1.0, phasecentre=phasecentre,
polarisation_frame=polarisation_frame,
zerow=zerow))
else:
raise NotImplementedError("order $s not known" % order)
return vis_list
if __name__ == '__main__':
log = logging.getLogger()
logging.info("Starting ICAL pipeline")
arlexecute.set_client(use_dask=True, threads_per_worker=1, memory_limit=32 * 1024 * 1024 * 1024, n_workers=8,
local_dir=dask_dir, verbose=True)
print(arlexecute.client)
arlexecute.run(init_logging)
nfreqwin = 41
ntimes = 5
rmax = 750.0
centre = nfreqwin // 2
# Load data from previous simulation
block_vislist = [arlexecute.execute(import_blockvisibility_from_hdf5)
(arl_path('%s/ska-pipeline_simulation_vislist_%d.hdf' % (results_dir, v)))
for v in range(nfreqwin)]
vis_list = [arlexecute.execute(convert_blockvisibility_to_visibility, nout=1)(bv) for bv in block_vislist]
print('Reading visibilities')
vis_list = arlexecute.compute(vis_list, sync=True)
cellsize = 0.001
npixel = 1024
pol_frame = PolarisationFrame("stokesI")
model_list = [arlexecute.execute(create_image_from_visibility)(v, npixel=npixel, cellsize=cellsize,
polarisation_frame=pol_frame)
for v in vis_list]
print('Creating model images')
def ical_list_arlexecute_workflow(vis_list,
model_imagelist,
context='2d',
calibration_context='TG',
do_selfcal=True,
**kwargs):
"""Create graph for ICAL pipeline
:param vis_list:
:param model_imagelist:
:param context: imaging context e.g. '2d'
:param calibration_context: Sequence of calibration steps e.g. TGB
:param do_selfcal: Do the selfcalibration?
:param kwargs: Parameters for functions in components
:return:
"""
psf_imagelist = invert_list_arlexecute_workflow(vis_list,
model_imagelist,
dopsf=True,
context=context,
**kwargs)
model_vislist = zero_list_arlexecute_workflow(vis_list)
model_vislist = predict_list_arlexecute_workflow(model_vislist,
model_imagelist,
context=context,
**kwargs)
if do_selfcal:
# Make the predicted visibilities, selfcalibrate against it correcting the gains, then
# form the residual visibility, then make the residual image
vis_list = calibrate_list_arlexecute_workflow(
vis_list,
model_vislist,
calibration_context=calibration_context,
**kwargs)
residual_vislist = subtract_list_arlexecute_workflow(
vis_list, model_vislist)
residual_imagelist = invert_list_arlexecute_workflow(residual_vislist,
model_imagelist,
dopsf=True,
context=context,
iteration=0,
**kwargs)
else:
# If we are not selfcalibrating it's much easier and we can avoid an unnecessary round of gather/scatter
# for visibility partitioning such as timeslices and wstack.
residual_imagelist = residual_list_arlexecute_workflow(vis_list,
model_imagelist,
context=context,
**kwargs)
deconvolve_model_imagelist, _ = deconvolve_list_arlexecute_workflow(
residual_imagelist,
psf_imagelist,
model_imagelist,
prefix='cycle 0',
**kwargs)
nmajor = get_parameter(kwargs, "nmajor", 5)
if nmajor > 1:
for cycle in range(nmajor):
if do_selfcal:
model_vislist = zero_list_arlexecute_workflow(vis_list)
model_vislist = predict_list_arlexecute_workflow(
model_vislist,
deconvolve_model_imagelist,
context=context,
**kwargs)
vis_list = calibrate_list_arlexecute_workflow(
vis_list,
model_vislist,
calibration_context=calibration_context,
iteration=cycle,
**kwargs)
residual_vislist = subtract_list_arlexecute_workflow(
vis_list, model_vislist)
residual_imagelist = invert_list_arlexecute_workflow(
residual_vislist,
model_imagelist,
dopsf=False,
context=context,
**kwargs)
else:
residual_imagelist = residual_list_arlexecute_workflow(
vis_list,
deconvolve_model_imagelist,
context=context,
**kwargs)
prefix = "cycle %d" % (cycle + 1)
deconvolve_model_imagelist, _ = deconvolve_list_arlexecute_workflow(
residual_imagelist,
psf_imagelist,
deconvolve_model_imagelist,
prefix=prefix,
**kwargs)
residual_imagelist = residual_list_arlexecute_workflow(
vis_list, deconvolve_model_imagelist, context=context, **kwargs)
restore_imagelist = restore_list_arlexecute_workflow(
deconvolve_model_imagelist, psf_imagelist, residual_imagelist)
return arlexecute.execute(
(deconvolve_model_imagelist, residual_imagelist, restore_imagelist))
def calibrate_list_arlexecute_workflow(vis_list,
model_vislist,
calibration_context='TG',
global_solution=True,
**kwargs):
""" Create a set of components for (optionally global) calibration of a list of visibilities
If global solution is true then visibilities are gathered to a single visibility data set which is then
self-calibrated. The resulting gaintable is then effectively scattered out for application to each visibility
set. If global solution is false then the solutions are performed locally.
:param vis_list:
:param model_vislist:
:param calibration_context: String giving terms to be calibrated e.g. 'TGB'
:param global_solution: Solve for global gains
:param kwargs: Parameters for functions in components
:return:
"""
def solve(vis, modelvis=None):
return solve_calibrate_function(
vis, modelvis, calibration_context=calibration_context, **kwargs)
def apply(vis, gt):
assert gt is not None
return apply_calibration_function(
vis, gt, calibration_context=calibration_context, **kwargs)
if global_solution:
point_vislist = [
arlexecute.execute(convert_visibility_to_blockvisibility,
nout=1)(v) for v in vis_list
]
point_modelvislist = [
arlexecute.execute(convert_visibility_to_blockvisibility,
nout=1)(mv) for mv in model_vislist
]
point_vislist = [
arlexecute.execute(divide_visibility,
nout=1)(point_vislist[i], point_modelvislist[i])
for i, _ in enumerate(point_vislist)
]
global_point_vis_list = arlexecute.execute(visibility_gather_channel,
nout=1)(point_vislist)
global_point_vis_list = arlexecute.execute(
integrate_visibility_by_channel, nout=1)(global_point_vis_list)
# This is a global solution so we only compute one gain table
gt_list = [
arlexecute.execute(solve, pure=True, nout=1)(global_point_vis_list)
]
return [
arlexecute.execute(apply, nout=1)(v, gt_list[0]) for v in vis_list
], gt_list
else:
gt_list = [
arlexecute.execute(solve, pure=True, nout=1)(v, model_vislist[i])
for i, v in enumerate(vis_list)
]
return [
arlexecute.execute(apply)(v, gt_list[i])
for i, v in enumerate(vis_list)
], gt_list
times = numpy.linspace(-300.0, 300.0, 3) * numpy.pi / 43200.0
nants = config.xyz.shape[0]
assert nants > 1
assert len(config.names) == nants
assert len(config.mount) == nants
config = create_named_configuration('LOWBD2', rmax=1000.0)
phasecentre = SkyCoord(ra=+15 * u.deg,
dec=-45.0 * u.deg,
frame='icrs',
equinox='J2000')
bvis_graph = arlexecute.execute(create_blockvisibility)(
config,
times,
frequency,
channel_bandwidth=channel_bandwidth,
phasecentre=phasecentre,
weight=1.0,
polarisation_frame=PolarisationFrame('stokesI'))
vis_graph = arlexecute.execute(convert_blockvisibility_to_visibility)(
bvis_graph)
model_graph = arlexecute.execute(create_image_from_visibility)(
vis_graph, npixel=4096, cellsize=0.001, override_cellsize=False)
beam = image_arlexecute_map_workflow(model_graph,
create_pb,
facets=16,
pointingcentre=phasecentre,
telescope='MID')
beam = arlexecute.compute(beam, sync=True)
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
bvis_list = simulate_list_arlexecute_workflow(
'LOWBD2',
frequency=frequency,
channel_bandwidth=channel_bandwidth,
times=times,
phasecentre=phasecentre,
order='frequency',
rmax=rmax)
print('%d elements in vis_list' % len(bvis_list))
log.info('About to make visibility')
vis_list = [
arlexecute.execute(convert_blockvisibility_to_visibility)(bv)
for bv in bvis_list
]
vis_list = arlexecute.compute(vis_list, sync=True)
# In[5]:
wprojection_planes = 1
advice_low = advise_wide_field(vis_list[0],
guard_band_image=8.0,
delA=0.02,
wprojection_planes=wprojection_planes)
advice_high = advise_wide_field(vis_list[-1],
guard_band_image=8.0,
delA=0.02,
vis_slices=vis_slices)[0]
c, resid = deconvolve_cube(d,
p,
m,
threshold=0.01,
fracthresh=0.01,
window_shape='quarter',
niter=100,
gain=0.1,
algorithm='hogbom-complex')
r = restore_cube(c, p, resid, psfwidth=psfwidth)
return r
print('About assemble cubes and deconvolve each frequency')
restored_list = [
arlexecute.execute(load_invert_and_deconvolve)(c) for c in range(nchan)
]
restored_cube = arlexecute.execute(image_gather_channels,
nout=1)(restored_list)
# restored_cube.visualize('dprepb_flat_arlexecute_pipeline.svg')
restored_cube = arlexecute.compute(restored_cube, sync=True)
print("Processing took %.3f s" % (time.time() - start))
print(qa_image(restored_cube, context='CLEAN restored cube'))
export_image_to_fits(
restored_cube, '%s/dprepb_arlexecute_%s_clean_restored_cube.fits' %
(results_dir, context))
try:
arlexecute.close()
except:
def continuum_imaging_list_arlexecute_workflow(vis_list,
model_imagelist,
context,
gcfcf=None,
vis_slices=1,
facets=1,
**kwargs):
""" Create graph for the continuum imaging pipeline.
Same as ICAL but with no selfcal.
:param vis_list:
:param model_imagelist:
:param context: Imaging context
:param kwargs: Parameters for functions in components
:return:
"""
if gcfcf is None:
gcfcf = [
arlexecute.execute(create_pswf_convolutionfunction)(
model_imagelist[0])
]
psf_imagelist = invert_list_arlexecute_workflow(vis_list,
model_imagelist,
context=context,
dopsf=True,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)
residual_imagelist = residual_list_arlexecute_workflow(
vis_list,
model_imagelist,
context=context,
gcfcf=gcfcf,
vis_slices=vis_slices,
facets=facets,
**kwargs)
deconvolve_model_imagelist = deconvolve_list_arlexecute_workflow(
residual_imagelist,
psf_imagelist,
model_imagelist,
prefix='cycle 0',
**kwargs)
nmajor = get_parameter(kwargs, "nmajor", 5)
if nmajor > 1:
for cycle in range(nmajor):
prefix = "cycle %d" % (cycle + 1)
residual_imagelist = residual_list_arlexecute_workflow(
vis_list,
deconvolve_model_imagelist,
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)
deconvolve_model_imagelist = deconvolve_list_arlexecute_workflow(
residual_imagelist,
psf_imagelist,
deconvolve_model_imagelist,
prefix=prefix,
**kwargs)
residual_imagelist = residual_list_arlexecute_workflow(
vis_list,
deconvolve_model_imagelist,
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)
restore_imagelist = restore_list_arlexecute_workflow(
deconvolve_model_imagelist, psf_imagelist, residual_imagelist)
return (deconvolve_model_imagelist, residual_imagelist, restore_imagelist)
def ical_list_arlexecute_workflow(vis_list,
model_imagelist,
context,
vis_slices=1,
facets=1,
gcfcf=None,
calibration_context='TG',
do_selfcal=True,
**kwargs):
"""Create graph for ICAL pipeline
:param vis_list:
:param model_imagelist:
:param context: imaging context e.g. '2d'
:param calibration_context: Sequence of calibration steps e.g. TGB
:param do_selfcal: Do the selfcalibration?
:param kwargs: Parameters for functions in components
:return:
"""
gt_list = list()
if gcfcf is None:
gcfcf = [
arlexecute.execute(create_pswf_convolutionfunction)(
model_imagelist[0])
]
psf_imagelist = invert_list_arlexecute_workflow(vis_list,
model_imagelist,
dopsf=True,
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)
model_vislist = [
arlexecute.execute(copy_visibility, nout=1)(v, zero=True)
for v in vis_list
]
if do_selfcal:
cal_vis_list = [
arlexecute.execute(copy_visibility, nout=1)(v) for v in vis_list
]
else:
cal_vis_list = vis_list
if do_selfcal:
# Make the predicted visibilities, selfcalibrate against it correcting the gains, then
# form the residual visibility, then make the residual image
predicted_model_vislist = predict_list_arlexecute_workflow(
model_vislist,
model_imagelist,
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)
recal_vis_list, gt_list = calibrate_list_arlexecute_workflow(
cal_vis_list,
predicted_model_vislist,
calibration_context=calibration_context,
**kwargs)
def zero_model_image(im):
log.info(
"ical_list_arlexecute_workflow: setting initial mode to zero after initial selfcal"
)
im.data[...] = 0.0
return im
model_imagelist = [
arlexecute.execute(zero_model_image, nout=1)(model)
for model in model_imagelist
]
residual_imagelist = invert_list_arlexecute_workflow(
recal_vis_list,
model_imagelist,
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)
else:
# If we are not selfcalibrating it's much easier and we can avoid an unnecessary round of gather/scatter
# for visibility partitioning such as timeslices and wstack.
residual_imagelist = residual_list_arlexecute_workflow(
cal_vis_list,
model_imagelist,
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)
deconvolve_model_imagelist = deconvolve_list_arlexecute_workflow(
residual_imagelist,
psf_imagelist,
model_imagelist,
prefix='cycle 0',
**kwargs)
nmajor = get_parameter(kwargs, "nmajor", 5)
if nmajor > 1:
for cycle in range(nmajor):
if do_selfcal:
predicted_model_vislist = predict_list_arlexecute_workflow(
model_vislist,
deconvolve_model_imagelist,
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)
recal_vis_list, gt_list = calibrate_list_arlexecute_workflow(
cal_vis_list,
predicted_model_vislist,
calibration_context=calibration_context,
iteration=cycle,
**kwargs)
residual_vislist = subtract_list_arlexecute_workflow(
recal_vis_list, model_vislist)
residual_imagelist = invert_list_arlexecute_workflow(
residual_vislist,
model_imagelist,
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)
else:
residual_imagelist = residual_list_arlexecute_workflow(
cal_vis_list,
deconvolve_model_imagelist,
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)
prefix = "cycle %d" % (cycle + 1)
deconvolve_model_imagelist = deconvolve_list_arlexecute_workflow(
residual_imagelist,
psf_imagelist,
deconvolve_model_imagelist,
prefix=prefix,
**kwargs)
residual_imagelist = residual_list_arlexecute_workflow(
cal_vis_list,
deconvolve_model_imagelist,
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)
restore_imagelist = restore_list_arlexecute_workflow(
deconvolve_model_imagelist, psf_imagelist, residual_imagelist)
return (deconvolve_model_imagelist, residual_imagelist, restore_imagelist,
gt_list)
def actualSetUp(self,
nfreqwin=3,
dospectral=True,
dopol=False,
amp_errors=None,
phase_errors=None,
zerow=True):
if amp_errors is None:
amp_errors = {'T': 0.0, 'G': 0.1}
if phase_errors is None:
phase_errors = {'T': 1.0, 'G': 0.0}
self.npixel = 512
self.low = create_named_configuration('LOWBD2', rmax=750.0)
self.freqwin = nfreqwin
self.vis_list = list()
self.ntimes = 1
self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
if self.freqwin > 1:
self.channelwidth = numpy.array(
self.freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.channelwidth = numpy.array([1e6])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
if dospectral:
flux = numpy.array(
[f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency])
else:
flux = numpy.array([f])
self.phasecentre = SkyCoord(ra=+180.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
self.blockvis_list = [
arlexecute.execute(ingest_unittest_visibility,
nout=1)(self.low, [self.frequency[i]],
[self.channelwidth[i]],
self.times,
self.vis_pol,
self.phasecentre,
block=True,
zerow=zerow) for i in range(nfreqwin)
]
self.blockvis_list = arlexecute.compute(self.blockvis_list, sync=True)
for v in self.blockvis_list:
v.data['vis'][...] = 1.0 + 0.0j
self.error_blockvis_list = [
arlexecute.execute(copy_visibility(v)) for v in self.blockvis_list
]
gt = arlexecute.execute(create_gaintable_from_blockvisibility)(
self.blockvis_list[0])
gt = arlexecute.execute(simulate_gaintable)(gt,
phase_error=0.1,
amplitude_error=0.0,
smooth_channels=1,
leakage=0.0,
seed=180555)
self.error_blockvis_list = [
arlexecute.execute(apply_gaintable)(self.error_blockvis_list[i],
gt)
for i in range(self.freqwin)
]
self.error_blockvis_list = arlexecute.compute(self.error_blockvis_list,
sync=True)
assert numpy.max(
numpy.abs(self.error_blockvis_list[0].vis -
self.blockvis_list[0].vis)) > 0.0
def trial_case(results,
seed=180555,
context='wstack',
nworkers=8,
threads_per_worker=1,
memory=8,
processes=True,
order='frequency',
nfreqwin=7,
ntimes=3,
rmax=750.0,
facets=1,
wprojection_planes=1,
use_dask=True,
use_serial=False):
""" Single trial for performance-timings
Simulates visibilities from GLEAM including phase errors
Makes dirty image and PSF
Runs ICAL pipeline
The results are in a dictionary:
'context': input - a string describing concisely the purpose of the test
'time overall', overall execution time (s)
'time create gleam', time to create GLEAM prediction graph
'time predict', time to execute GLEAM prediction graph
'time corrupt', time to corrupt data_models
'time invert', time to make dirty image
'time psf invert', time to make PSF
'time ICAL graph', time to create ICAL graph
'time ICAL', time to execute ICAL graph
'context', type of imaging e.g. 'wstack'
'nworkers', number of workers to create
'threads_per_worker',
'nnodes', Number of nodes,
'processes', 'order', Ordering of data_models
'nfreqwin', Number of frequency windows in simulation
'ntimes', Number of hour angles in simulation
'rmax', Maximum radius of stations used in simulation (m)
'facets', Number of facets in deconvolution and imaging
'wprojection_planes', Number of wprojection planes
'vis_slices', Number of visibility slices (per Visibbility)
'npixel', Number of pixels in image
'cellsize', Cellsize in radians
'seed', Random number seed
'dirty_max', Maximum in dirty image
'dirty_min', Minimum in dirty image
'psf_max',
'psf_min',
'restored_max',
'restored_min',
'deconvolved_max',
'deconvolved_min',
'residual_max',
'residual_min',
'git_info', GIT hash (not definitive since local mods are possible)
:param results: Initial state
:param seed: Random number seed (used in gain simulations)
:param context: imaging context
:param context: Type of context: '2d'|'timeslice'|'wstack'
:param nworkers: Number of dask workers to use
:param threads_per_worker: Number of threads per worker
:param processes: Use processes instead of threads 'processes'|'threads'
:param order: See simulate_list_list_arlexecute_workflow_workflowkflow
:param nfreqwin: See simulate_list_list_arlexecute_workflow_workflowkflow
:param ntimes: See simulate_list_list_arlexecute_workflow_workflowkflow
:param rmax: See simulate_list_list_arlexecute_workflow_workflowkflow
:param facets: Number of facets to use
:param wprojection_planes: Number of wprojection planes to use
:param use_dask: Use dask or immediate evaluation
:param kwargs:
:return: results dictionary
"""
numpy.random.seed(seed)
results['seed'] = seed
start_all = time.time()
results['context'] = context
results['hostname'] = socket.gethostname()
results['git_hash'] = git_hash()
results['epoch'] = time.strftime("%Y-%m-%d %H:%M:%S")
zerow = False
print("Context is %s" % context)
results['nworkers'] = nworkers
results['threads_per_worker'] = threads_per_worker
results['processes'] = processes
results['memory'] = memory
results['order'] = order
results['nfreqwin'] = nfreqwin
results['ntimes'] = ntimes
results['rmax'] = rmax
results['facets'] = facets
results['wprojection_planes'] = wprojection_planes
results['use_dask'] = use_dask
print("At start, configuration is {0!r}".format(results))
# Parameters determining scale
frequency = numpy.linspace(0.8e8, 1.2e8, nfreqwin)
centre = nfreqwin // 2
if nfreqwin > 1:
channel_bandwidth = numpy.array(nfreqwin *
[frequency[1] - frequency[0]])
else:
channel_bandwidth = numpy.array([1e6])
times = numpy.linspace(-numpy.pi / 3.0, numpy.pi / 3.0, ntimes)
phasecentre = SkyCoord(ra=+30.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
if use_dask:
client = get_dask_Client(threads_per_worker=threads_per_worker,
memory_limit=memory * 1024 * 1024 * 1024,
n_workers=nworkers)
arlexecute.set_client(client)
nodes = findNodes(arlexecute.client)
unodes = list(numpy.unique(nodes))
results['nnodes'] = len(unodes)
print("Defined %d workers on %d nodes" % (nworkers, results['nnodes']))
print("Workers are: %s" % str(nodes))
else:
arlexecute.set_client(use_dask=use_dask)
results['nnodes'] = 1
unodes = None
vis_list = simulate_list_arlexecute_workflow(
'LOWBD2',
frequency=frequency,
channel_bandwidth=channel_bandwidth,
times=times,
phasecentre=phasecentre,
order=order,
format='blockvis',
rmax=rmax)
print("****** Visibility creation ******")
vis_list = arlexecute.persist(vis_list)
# Find the best imaging parameters but don't bring the vis_list back here
def get_wf(bv):
v = convert_blockvisibility_to_visibility(bv)
return advise_wide_field(v,
guard_band_image=6.0,
delA=0.02,
facets=facets,
wprojection_planes=wprojection_planes,
oversampling_synthesised_beam=4.0)
wprojection_planes = 1
advice = arlexecute.compute(arlexecute.execute(get_wf)(vis_list[0]),
sync=True)
npixel = advice['npixels2']
cellsize = advice['cellsize']
if context == 'timeslice':
vis_slices = ntimes
print("Using timeslice with %d slices" % vis_slices)
elif context == '2d':
vis_slices = 1
else:
context = 'wstack'
vis_slices = 5 * advice['vis_slices']
print("Using wstack with %d slices" % vis_slices)
results['vis_slices'] = vis_slices
results['cellsize'] = cellsize
results['npixel'] = npixel
gleam_model_list = [
arlexecute.execute(create_low_test_image_from_gleam)(
npixel=npixel,
frequency=[frequency[f]],
channel_bandwidth=[channel_bandwidth[f]],
cellsize=cellsize,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
flux_limit=0.3,
applybeam=True) for f, freq in enumerate(frequency)
]
start = time.time()
print("****** Starting GLEAM model creation ******")
gleam_model_list = arlexecute.compute(gleam_model_list, sync=True)
cmodel = smooth_image(gleam_model_list[centre])
export_image_to_fits(cmodel,
"pipelines-timings-arlexecute-gleam_cmodel.fits")
end = time.time()
results['time create gleam'] = end - start
print("Creating GLEAM model took %.2f seconds" % (end - start))
gleam_model_list = arlexecute.scatter(gleam_model_list)
vis_list = predict_list_arlexecute_workflow(vis_list,
gleam_model_list,
vis_slices=vis_slices,
context=context)
start = time.time()
print("****** Starting GLEAM model visibility prediction ******")
vis_list = arlexecute.compute(vis_list, sync=True)
end = time.time()
results['time predict'] = end - start
print("GLEAM model Visibility prediction took %.2f seconds" %
(end - start))
# Corrupt the visibility for the GLEAM model
print("****** Visibility corruption ******")
vis_list = corrupt_list_arlexecute_workflow(vis_list, phase_error=1.0)
start = time.time()
vis_list = arlexecute.compute(vis_list, sync=True)
vis_list = arlexecute.scatter(vis_list)
end = time.time()
results['time corrupt'] = end - start
print("Visibility corruption took %.2f seconds" % (end - start))
# Create an empty model image
model_list = [
arlexecute.execute(create_image_from_visibility)(
vis_list[f],
npixel=npixel,
cellsize=cellsize,
frequency=[frequency[f]],
channel_bandwidth=[channel_bandwidth[f]],
polarisation_frame=PolarisationFrame("stokesI"))
for f, freq in enumerate(frequency)
]
model_list = arlexecute.compute(model_list, sync=True)
model_list = arlexecute.scatter(model_list)
psf_list = invert_list_arlexecute_workflow(vis_list,
model_list,
vis_slices=vis_slices,
context=context,
facets=facets,
dopsf=True)
start = time.time()
print("****** Starting PSF calculation ******")
psf, sumwt = arlexecute.compute(psf_list, sync=True)[centre]
end = time.time()
results['time psf invert'] = end - start
print("PSF invert took %.2f seconds" % (end - start))
results['psf_max'] = qa_image(psf).data['max']
results['psf_min'] = qa_image(psf).data['min']
dirty_list = invert_list_arlexecute_workflow(vis_list,
model_list,
vis_slices=vis_slices,
context=context,
facets=facets)
start = time.time()
print("****** Starting dirty image calculation ******")
dirty, sumwt = arlexecute.compute(dirty_list, sync=True)[centre]
end = time.time()
results['time invert'] = end - start
print("Dirty image invert took %.2f seconds" % (end - start))
print("Maximum in dirty image is ", numpy.max(numpy.abs(dirty.data)),
", sumwt is ", sumwt)
qa = qa_image(dirty)
results['dirty_max'] = qa.data['max']
results['dirty_min'] = qa.data['min']
# Create the ICAL pipeline to run 5 major cycles, starting selfcal at cycle 1. A global solution across all
# frequencies (i.e. Visibilities) is performed.
start = time.time()
print("****** Starting ICAL ******")
controls = create_calibration_controls()
controls['T']['first_selfcal'] = 1
controls['G']['first_selfcal'] = 3
controls['B']['first_selfcal'] = 4
controls['T']['timescale'] = 'auto'
controls['G']['timescale'] = 'auto'
controls['B']['timescale'] = 1e5
if nfreqwin > 6:
nmoment = 3
algorithm = 'mmclean'
elif nfreqwin > 2:
nmoment = 2
algorithm = 'mmclean'
else:
nmoment = 1
algorithm = 'msclean'
start = time.time()
ical_list = ical_list_arlexecute_workflow(vis_list,
model_imagelist=model_list,
context='wstack',
calibration_context='TG',
controls=controls,
scales=[0, 3, 10],
algorithm=algorithm,
nmoment=nmoment,
niter=1000,
fractional_threshold=0.1,
threshold=0.1,
nmajor=5,
gain=0.25,
vis_slices=vis_slices,
timeslice='auto',
global_solution=False,
psf_support=64,
do_selfcal=True)
end = time.time()
results['time ICAL graph'] = end - start
print("Construction of ICAL graph took %.2f seconds" % (end - start))
# Execute the graph
start = time.time()
result = arlexecute.compute(ical_list, sync=True)
deconvolved, residual, restored = result
end = time.time()
results['time ICAL'] = end - start
print("ICAL graph execution took %.2f seconds" % (end - start))
qa = qa_image(deconvolved[centre])
results['deconvolved_max'] = qa.data['max']
results['deconvolved_min'] = qa.data['min']
export_image_to_fits(deconvolved[centre],
"pipelines-timings-arlexecute-ical_deconvolved.fits")
qa = qa_image(residual[centre][0])
results['residual_max'] = qa.data['max']
results['residual_min'] = qa.data['min']
export_image_to_fits(residual[centre][0],
"pipelines-timings-arlexecute-ical_residual.fits")
qa = qa_image(restored[centre])
results['restored_max'] = qa.data['max']
results['restored_min'] = qa.data['min']
export_image_to_fits(restored[centre],
"pipelines-timings-arlexecute-ical_restored.fits")
#
arlexecute.close()
end_all = time.time()
results['time overall'] = end_all - start_all
print("At end, results are {0!r}".format(results))
return results
def predict_list_arlexecute_workflow(vis_list,
model_imagelist,
vis_slices=1,
facets=1,
context='2d',
**kwargs):
"""Predict, iterating over both the scattered vis_list and image
The visibility and image are scattered, the visibility is predicted on each part, and then the
parts are assembled.
:param vis_list:
:param model_imagelist: Model used to determine image parameters
:param vis_slices: Number of vis slices (w stack or timeslice)
:param facets: Number of facets (per axis)
:param context:
:param kwargs: Parameters for functions in components
:return: List of vis_lists
"""
assert len(vis_list) == len(
model_imagelist), "Model must be the same length as the vis_list"
c = imaging_context(context)
vis_iter = c['vis_iterator']
predict = c['predict']
if facets % 2 == 0 or facets == 1:
actual_number_facets = facets
else:
actual_number_facets = facets - 1
def predict_ignore_none(vis, model):
if vis is not None:
return predict(vis,
model,
context=context,
facets=facets,
vis_slices=vis_slices,
**kwargs)
else:
return None
image_results_list_list = list()
# Loop over all frequency windows
for freqwin, vis_list in enumerate(vis_list):
# Create the graph to divide an image into facets. This is by reference.
facet_lists = arlexecute.execute(image_scatter_facets,
nout=actual_number_facets**2)(
model_imagelist[freqwin],
facets=facets)
# Create the graph to divide the visibility into slices. This is by copy.
sub_vis_lists = arlexecute.execute(visibility_scatter,
nout=vis_slices)(vis_list, vis_iter,
vis_slices)
facet_vis_lists = list()
# Loop over sub visibility
for sub_vis_list in sub_vis_lists:
facet_vis_results = list()
# Loop over facets
for facet_list in facet_lists:
# Predict visibility for this subvisibility from this facet
facet_vis_list = arlexecute.execute(predict_ignore_none,
pure=True,
nout=1)(sub_vis_list,
facet_list)
facet_vis_results.append(facet_vis_list)
# Sum the current sub-visibility over all facets
facet_vis_lists.append(
arlexecute.execute(sum_predict_results)(facet_vis_results))
# Sum all sub-visibilties
image_results_list_list.append(
arlexecute.execute(visibility_gather, nout=1)(facet_vis_lists,
vis_list, vis_iter))
return image_results_list_list
def actualSetUp(self, freqwin=1, block=True, dopol=False, zerow=False):
self.npixel = 1024
self.low = create_named_configuration('LOWBD2', rmax=550.0)
self.freqwin = freqwin
self.blockvis_list = list()
self.ntimes = 5
self.cellsize = 0.0005
# Choose the interval so that the maximum change in w is smallish
integration_time = numpy.pi * (24 / (12 * 60))
self.times = numpy.linspace(-integration_time * (self.ntimes // 2),
integration_time * (self.ntimes // 2),
self.ntimes)
if freqwin > 1:
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
self.channelwidth = numpy.array(
freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.frequency = numpy.array([1.0e8])
self.channelwidth = numpy.array([4e7])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
self.phasecentre = SkyCoord(ra=+0.0 * u.deg,
dec=-40.0 * u.deg,
frame='icrs',
equinox='J2000')
self.blockvis_list = [
arlexecute.execute(ingest_unittest_visibility)(
self.low, [self.frequency[freqwin]],
[self.channelwidth[freqwin]],
self.times,
self.vis_pol,
self.phasecentre,
block=block,
zerow=zerow) for freqwin, _ in enumerate(self.frequency)
]
self.blockvis_list = arlexecute.compute(self.blockvis_list, sync=True)
self.vis_list = [
arlexecute.execute(convert_blockvisibility_to_visibility)(bv)
for bv in self.blockvis_list
]
self.vis_list = arlexecute.compute(self.vis_list, sync=True)
self.skymodel_list = [
arlexecute.execute(create_low_test_skymodel_from_gleam)(
npixel=self.npixel,
cellsize=self.cellsize,
frequency=[self.frequency[f]],
phasecentre=self.phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
flux_limit=0.6,
flux_threshold=1.0,
flux_max=5.0) for f, freq in enumerate(self.frequency)
]
self.skymodel_list = arlexecute.compute(self.skymodel_list, sync=True)
assert isinstance(self.skymodel_list[0].image,
Image), self.skymodel_list[0].image
assert isinstance(self.skymodel_list[0].components[0],
Skycomponent), self.skymodel_list[0].components[0]
assert len(self.skymodel_list[0].components) == 35, len(
self.skymodel_list[0].components)
self.skymodel_list = expand_skymodel_by_skycomponents(
self.skymodel_list[0])
assert len(self.skymodel_list) == 36, len(self.skymodel_list)
assert numpy.max(numpy.abs(
self.skymodel_list[-1].image.data)) > 0.0, "Image is empty"
self.vis_list = [
copy_visibility(self.vis_list[0], zero=True)
for i, _ in enumerate(self.skymodel_list)
]
'LOWBD2',
rmax=rmax,
frequency=frequency,
channel_bandwidth=channel_bandwidth,
times=times,
phasecentre=phasecentre,
order='frequency')
print('%d elements in vis_list' % len(vis_list))
print('About to make visibility')
vis_list = arlexecute.persist(vis_list)
# The vis data are on the workers so we run the advice function on the workers
# without transfering the data back to the host.
advice_list = [
arlexecute.execute(advise_wide_field)(v,
guard_band_image=8.0,
delA=0.02,
wprojection_planes=1)
for _, v in enumerate(vis_list)
]
advice_list = arlexecute.compute(advice_list, sync=True)
advice_low = advice_list[0]
advice_high = advice_list[-1]
print(advice_list[centre])
vis_slices = advice_low['vis_slices']
npixel = advice_high['npixels2']
cellsize = min(advice_low['cellsize'], advice_high['cellsize'])
# Now make a graph to fill with a model drawn from GLEAM
def trial_case(results,
seed=180555,
context='wstack',
nworkers=8,
threads_per_worker=1,
memory=8,
processes=True,
order='frequency',
nfreqwin=7,
ntimes=3,
rmax=750.0,
facets=1,
wprojection_planes=1,
use_dask=True,
use_serial_imaging=True,
flux_limit=0.3,
nmajor=5,
dft_threshold=1.0,
use_serial_clean=True,
write_fits=False):
""" Single trial for performance-timings
Simulates visibilities from GLEAM including phase errors
Makes dirty image and PSF
Runs ICAL pipeline
The results are in a dictionary:
'context': input - a string describing concisely the purpose of the test
'time overall', overall execution time (s)
'time predict', time to execute GLEAM prediction graph
'time invert', time to make dirty image
'time invert graph', time to make dirty image graph
'time ICAL graph', time to create ICAL graph
'time ICAL', time to execute ICAL graph
'context', type of imaging e.g. 'wstack'
'nworkers', number of workers to create
'threads_per_worker',
'nnodes', Number of nodes,
'processes', 'order', Ordering of data_models
'nfreqwin', Number of frequency windows in simulation
'ntimes', Number of hour angles in simulation
'rmax', Maximum radius of stations used in simulation (m)
'facets', Number of facets in deconvolution and imaging
'wprojection_planes', Number of wprojection planes
'vis_slices', Number of visibility slices (per Visibbility)
'npixel', Number of pixels in image
'cellsize', Cellsize in radians
'seed', Random number seed
'dirty_max', Maximum in dirty image
'dirty_min', Minimum in dirty image
'restored_max',
'restored_min',
'deconvolved_max',
'deconvolved_min',
'residual_max',
'residual_min',
'git_info', GIT hash (not definitive since local mods are possible)
:param results: Initial state
:param seed: Random number seed (used in gain simulations)
:param context: imaging context
:param context: Type of context: '2d'|'timeslice'|'wstack'
:param nworkers: Number of dask workers to use
:param threads_per_worker: Number of threads per worker
:param processes: Use processes instead of threads 'processes'|'threads'
:param order: See simulate_list_list_arlexecute_workflow_workflowkflow
:param nfreqwin: See simulate_list_list_arlexecute_workflow_workflowkflow
:param ntimes: See simulate_list_list_arlexecute_workflow_workflowkflow
:param rmax: See simulate_list_list_arlexecute_workflow_workflowkflow
:param facets: Number of facets to use
:param wprojection_planes: Number of wprojection planes to use
:param use_dask: Use dask or immediate evaluation
:return: results dictionary
"""
if use_dask:
client = get_dask_Client(threads_per_worker=threads_per_worker,
processes=threads_per_worker == 1,
memory_limit=memory * 1024 * 1024 * 1024,
n_workers=nworkers)
arlexecute.set_client(client)
nodes = findNodes(arlexecute.client)
print("Defined %d workers on %d nodes" % (nworkers, len(nodes)))
print("Workers are: %s" % str(nodes))
else:
arlexecute.set_client(use_dask=use_dask)
results['nnodes'] = 1
def init_logging():
logging.basicConfig(
filename='pipelines_arlexecute_timings.log',
filemode='w',
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
init_logging()
log = logging.getLogger()
# Initialise logging on the workers. This appears to only work using the process scheduler.
arlexecute.run(init_logging)
def lprint(*args):
log.info(*args)
print(*args)
lprint("Starting pipelines_arlexecute_timings")
numpy.random.seed(seed)
results['seed'] = seed
start_all = time.time()
results['context'] = context
results['hostname'] = socket.gethostname()
results['git_hash'] = git_hash()
results['epoch'] = time.strftime("%Y-%m-%d %H:%M:%S")
lprint("Context is %s" % context)
results['nworkers'] = nworkers
results['threads_per_worker'] = threads_per_worker
results['processes'] = processes
results['memory'] = memory
results['order'] = order
results['nfreqwin'] = nfreqwin
results['ntimes'] = ntimes
results['rmax'] = rmax
results['facets'] = facets
results['wprojection_planes'] = wprojection_planes
results['dft threshold'] = dft_threshold
results['use_dask'] = use_dask
lprint("At start, configuration is:")
lprint(sort_dict(results))
# Parameters determining scale of simulation.
frequency = numpy.linspace(1.0e8, 1.2e8, nfreqwin)
centre = nfreqwin // 2
if nfreqwin > 1:
channel_bandwidth = numpy.array(nfreqwin *
[frequency[1] - frequency[0]])
else:
channel_bandwidth = numpy.array([1e6])
times = numpy.linspace(-numpy.pi / 4.0, numpy.pi / 4.0, ntimes)
phasecentre = SkyCoord(ra=+0.0 * u.deg,
dec=-40.0 * u.deg,
frame='icrs',
equinox='J2000')
lprint("****** Visibility creation ******")
# Create the empty BlockVisibility's and persist these on the cluster
tmp_bvis_list = simulate_list_arlexecute_workflow(
'LOWBD2',
frequency=frequency,
channel_bandwidth=channel_bandwidth,
times=times,
phasecentre=phasecentre,
order=order,
format='blockvis',
rmax=rmax)
tmp_vis_list = [
arlexecute.execute(convert_blockvisibility_to_visibility)(bv)
for bv in tmp_bvis_list
]
tmp_vis_list = arlexecute.client.compute(tmp_vis_list, sync=True)
vis_list = arlexecute.gather(tmp_vis_list)
import matplotlib.pyplot as plt
plt.clf()
plt.hist(vis_list[0].w, bins=100)
plt.title('Histogram of w samples: rms=%.1f (wavelengths)' %
numpy.std(vis_list[0].w))
plt.xlabel('W (wavelengths)')
plt.show()
plt.clf()
plt.hist(vis_list[0].uvdist, bins=100)
plt.title('Histogram of uvdistance samples')
plt.xlabel('UV Distance (wavelengths)')
plt.show()
arlexecute.client.cancel(tmp_vis_list)
future_vis_list = arlexecute.scatter(vis_list)
# Find the best imaging parameters but don't bring the vis_list back here
print("****** Finding wide field parameters ******")
future_advice = [
arlexecute.execute(advise_wide_field)(
v,
guard_band_image=6.0,
delA=0.1,
facets=facets,
wprojection_planes=wprojection_planes,
oversampling_synthesised_beam=4.0) for v in future_vis_list
]
future_advice = arlexecute.compute(future_advice)
advice = arlexecute.client.gather(future_advice)[-1]
arlexecute.client.cancel(future_advice)
# Deconvolution via sub-images requires 2^n
npixel = advice['npixels2']
results['npixel'] = npixel
cellsize = advice['cellsize']
results['cellsize'] = cellsize
lprint("Image will have %d by %d pixels, cellsize = %.6f rad" %
(npixel, npixel, cellsize))
# Create an empty model image
tmp_model_list = [
arlexecute.execute(create_image)(
npixel=npixel,
cellsize=cellsize,
frequency=[frequency[f]],
channel_bandwidth=[channel_bandwidth[f]],
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
for f, freq in enumerate(frequency)
]
model_list = arlexecute.compute(tmp_model_list, sync=True)
future_model_list = arlexecute.scatter(model_list)
lprint("****** Setting up imaging parameters ******")
# Now set up the imaging parameters
template_model = create_image(
npixel=npixel,
cellsize=cellsize,
frequency=[frequency[centre]],
phasecentre=phasecentre,
channel_bandwidth=[channel_bandwidth[centre]],
polarisation_frame=PolarisationFrame("stokesI"))
gcfcf = [create_pswf_convolutionfunction(template_model)]
if context == 'timeslice':
vis_slices = ntimes
lprint("Using timeslice with %d slices" % vis_slices)
elif context == '2d':
vis_slices = 1
elif context == "wprojection":
wstep = advice['wstep']
nw = advice['wprojection_planes']
vis_slices = 1
support = advice['nwpixels']
results['wprojection_planes'] = nw
lprint("****** Starting W projection kernel creation ******")
lprint("Using wprojection with %d planes with wstep %.1f wavelengths" %
(nw, wstep))
lprint("Support of wprojection = %d pixels" % support)
gcfcf = [
create_awterm_convolutionfunction(template_model,
nw=nw,
wstep=wstep,
oversampling=4,
support=support,
use_aaf=True)
]
lprint("Size of W projection gcf, cf = %.2E bytes" % get_size(gcfcf))
else:
context = 'wstack'
vis_slices = advice['vis_slices']
lprint("Using wstack with %d slices" % vis_slices)
gcfcf = arlexecute.scatter(gcfcf, broadcast=True)
results['vis_slices'] = vis_slices
# Make a skymodel from gleam, with bright sources as components and weak sources in an image
lprint("****** Starting GLEAM skymodel creation ******")
future_skymodel_list = [
arlexecute.execute(create_low_test_skymodel_from_gleam)(
npixel=npixel,
cellsize=cellsize,
frequency=[frequency[f]],
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
flux_limit=flux_limit,
flux_threshold=dft_threshold,
flux_max=5.0) for f, freq in enumerate(frequency)
]
# We use predict_skymodel so that we can use skycomponents as well as images
lprint("****** Starting GLEAM skymodel prediction ******")
predicted_vis_list = [
predict_skymodel_list_arlexecute_workflow(future_vis_list[f],
[future_skymodel_list[f]],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf)[0]
for f, freq in enumerate(frequency)
]
# Corrupt the visibility for the GLEAM model
lprint("****** Visibility corruption ******")
tmp_corrupted_vis_list = corrupt_list_arlexecute_workflow(
predicted_vis_list, phase_error=1.0, seed=seed)
lprint("****** Weighting and tapering ******")
tmp_corrupted_vis_list = weight_list_arlexecute_workflow(
tmp_corrupted_vis_list, future_model_list)
tmp_corrupted_vis_list = taper_list_arlexecute_workflow(
tmp_corrupted_vis_list, 0.003 * 750.0 / rmax)
tmp_corrupted_vis_list = arlexecute.compute(tmp_corrupted_vis_list,
sync=True)
corrupted_vis_list = arlexecute.gather(tmp_corrupted_vis_list)
# arlexecute.client.cancel(predicted_vis_list)
arlexecute.client.cancel(tmp_corrupted_vis_list)
future_corrupted_vis_list = arlexecute.scatter(corrupted_vis_list)
# At this point the only futures are of scatter'ed data so no repeated calculations should be
# incurred.
lprint("****** Starting dirty image calculation ******")
start = time.time()
dirty_list = invert_list_arlexecute_workflow(
future_corrupted_vis_list,
future_model_list,
vis_slices=vis_slices,
context=context,
facets=facets,
use_serial_invert=use_serial_imaging,
gcfcf=gcfcf)
results['size invert graph'] = get_size(dirty_list)
lprint('Size of dirty graph is %.3E bytes' %
(results['size invert graph']))
end = time.time()
results['time invert graph'] = end - start
lprint("Construction of invert graph took %.3f seconds" % (end - start))
start = time.time()
dirty, sumwt = arlexecute.compute(dirty_list, sync=True)[centre]
end = time.time()
results['time invert'] = end - start
lprint("Dirty image invert took %.3f seconds" % (end - start))
lprint("Maximum in dirty image is %f, sumwt is %s" %
(numpy.max(numpy.abs(dirty.data)), str(sumwt)))
qa = qa_image(dirty)
results['dirty_max'] = qa.data['max']
results['dirty_min'] = qa.data['min']
if write_fits:
export_image_to_fits(
dirty, "pipelines_arlexecute_timings-%s-dirty.fits" % context)
lprint("****** Starting prediction ******")
start = time.time()
tmp_vis_list = predict_list_arlexecute_workflow(
future_corrupted_vis_list,
future_model_list,
vis_slices=vis_slices,
context=context,
facets=facets,
use_serial_predict=use_serial_imaging,
gcfcf=gcfcf)
result = arlexecute.compute(tmp_vis_list, sync=True)
# arlexecute.client.cancel(tmp_vis_list)
end = time.time()
results['time predict'] = end - start
lprint("Predict took %.3f seconds" % (end - start))
# Create the ICAL pipeline to run major cycles, starting selfcal at cycle 1. A global solution across all
# frequencies (i.e. Visibilities) is performed.
print("Using subimage clean")
deconvolve_facets = 8
deconvolve_overlap = 16
deconvolve_taper = 'tukey'
lprint("****** Starting ICAL graph creation ******")
controls = create_calibration_controls()
controls['T']['first_selfcal'] = 1
controls['T']['timeslice'] = 'auto'
start = time.time()
ical_list = ical_list_arlexecute_workflow(
future_corrupted_vis_list,
model_imagelist=future_model_list,
context=context,
vis_slices=vis_slices,
scales=[0, 3, 10],
algorithm='mmclean',
nmoment=3,
niter=1000,
fractional_threshold=0.1,
threshold=0.01,
nmajor=nmajor,
gain=0.25,
psf_support=64,
deconvolve_facets=deconvolve_facets,
deconvolve_overlap=deconvolve_overlap,
deconvolve_taper=deconvolve_taper,
timeslice='auto',
global_solution=True,
do_selfcal=True,
calibration_context='T',
controls=controls,
use_serial_predict=use_serial_imaging,
use_serial_invert=use_serial_imaging,
use_serial_clean=use_serial_clean,
gcfcf=gcfcf)
results['size ICAL graph'] = get_size(ical_list)
lprint('Size of ICAL graph is %.3E bytes' % results['size ICAL graph'])
end = time.time()
results['time ICAL graph'] = end - start
lprint("Construction of ICAL graph took %.3f seconds" % (end - start))
print("Current objects on cluster: ")
pp.pprint(arlexecute.client.who_has())
#
# Execute the graph
lprint("****** Executing ICAL graph ******")
start = time.time()
deconvolved, residual, restored, gaintables = arlexecute.compute(ical_list,
sync=True)
end = time.time()
results['time ICAL'] = end - start
lprint("ICAL graph execution took %.3f seconds" % (end - start))
qa = qa_image(deconvolved[centre])
results['deconvolved_max'] = qa.data['max']
results['deconvolved_min'] = qa.data['min']
deconvolved_cube = image_gather_channels(deconvolved)
if write_fits:
export_image_to_fits(
deconvolved_cube,
"pipelines_arlexecute_timings-%s-ical_deconvolved.fits" % context)
qa = qa_image(residual[centre][0])
results['residual_max'] = qa.data['max']
results['residual_min'] = qa.data['min']
residual_cube = remove_sumwt(residual)
residual_cube = image_gather_channels(residual_cube)
if write_fits:
export_image_to_fits(
residual_cube,
"pipelines_arlexecute_timings-%s-ical_residual.fits" % context)
qa = qa_image(restored[centre])
results['restored_max'] = qa.data['max']
results['restored_min'] = qa.data['min']
restored_cube = image_gather_channels(restored)
if write_fits:
export_image_to_fits(
restored_cube,
"pipelines_arlexecute_timings-%s-ical_restored.fits" % context)
#
arlexecute.close()
end_all = time.time()
results['time overall'] = end_all - start_all
lprint("At end, results are:")
results = sort_dict(results)
lprint(results)
return results
def actualSetUp(self,
add_errors=False,
nfreqwin=7,
dospectral=True,
dopol=False,
zerow=True):
self.npixel = 512
self.low = create_named_configuration('LOWBD2', rmax=750.0)
self.freqwin = nfreqwin
self.vis_list = list()
self.ntimes = 5
self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
if self.freqwin > 1:
self.channelwidth = numpy.array(
self.freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.channelwidth = numpy.array([1e6])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
if dospectral:
flux = numpy.array(
[f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency])
else:
flux = numpy.array([f])
self.phasecentre = SkyCoord(ra=+180.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
self.blockvis_list = [
arlexecute.execute(ingest_unittest_visibility,
nout=1)(self.low, [self.frequency[i]],
[self.channelwidth[i]],
self.times,
self.vis_pol,
self.phasecentre,
block=True,
zerow=zerow) for i in range(nfreqwin)
]
self.blockvis_list = arlexecute.compute(self.blockvis_list, sync=True)
self.blockvis_list = arlexecute.scatter(self.blockvis_list)
self.vis_list = [
arlexecute.execute(convert_blockvisibility_to_visibility,
nout=1)(bv) for bv in self.blockvis_list
]
self.vis_list = arlexecute.compute(self.vis_list, sync=True)
self.vis_list = arlexecute.scatter(self.vis_list)
self.model_imagelist = [
arlexecute.execute(create_unittest_model,
nout=1)(self.vis_list[i],
self.image_pol,
npixel=self.npixel,
cellsize=0.0005)
for i in range(nfreqwin)
]
self.model_imagelist = arlexecute.compute(self.model_imagelist,
sync=True)
self.model_imagelist = arlexecute.scatter(self.model_imagelist)
self.components_list = [
arlexecute.execute(create_unittest_components)(
self.model_imagelist[freqwin],
flux[freqwin, :][numpy.newaxis, :])
for freqwin, m in enumerate(self.model_imagelist)
]
self.components_list = arlexecute.compute(self.components_list,
sync=True)
self.components_list = arlexecute.scatter(self.components_list)
self.blockvis_list = [
arlexecute.execute(predict_skycomponent_visibility)(
self.blockvis_list[freqwin], self.components_list[freqwin])
for freqwin, _ in enumerate(self.blockvis_list)
]
self.blockvis_list = arlexecute.compute(self.blockvis_list, sync=True)
self.vis = self.blockvis_list[0]
self.blockvis_list = arlexecute.scatter(self.blockvis_list)
self.model_imagelist = [
arlexecute.execute(insert_skycomponent,
nout=1)(self.model_imagelist[freqwin],
self.components_list[freqwin])
for freqwin in range(nfreqwin)
]
self.model_imagelist = arlexecute.compute(self.model_imagelist,
sync=True)
model = self.model_imagelist[0]
self.cmodel = smooth_image(model)
if self.persist:
export_image_to_fits(
model, '%s/test_pipelines_arlexecute_model.fits' % self.dir)
export_image_to_fits(
self.cmodel,
'%s/test_pipelines_arlexecute_cmodel.fits' % self.dir)
if add_errors:
gt = create_gaintable_from_blockvisibility(self.vis)
gt = simulate_gaintable(gt,
phase_error=0.1,
amplitude_error=0.0,
smooth_channels=1,
leakage=0.0,
seed=180555)
self.blockvis_list = [
arlexecute.execute(apply_gaintable,
nout=1)(self.blockvis_list[i], gt)
for i in range(self.freqwin)
]
self.blockvis_list = arlexecute.compute(self.blockvis_list,
sync=True)
self.blockvis_list = arlexecute.scatter(self.blockvis_list)
self.vis_list = [
arlexecute.execute(convert_blockvisibility_to_visibility)(bv)
for bv in self.blockvis_list
]
self.vis_list = arlexecute.compute(self.vis_list, sync=True)
self.vis_list = arlexecute.scatter(self.vis_list)
self.model_imagelist = [
arlexecute.execute(create_unittest_model,
nout=1)(self.vis_list[i],
self.image_pol,
npixel=self.npixel,
cellsize=0.0005)
for i in range(nfreqwin)
]
self.model_imagelist = arlexecute.compute(self.model_imagelist,
sync=True)
self.model_imagelist = arlexecute.scatter(self.model_imagelist)
def deconvolve_list_arlexecute_workflow(dirty_list, psf_list, model_imagelist, prefix='', mask=None, **kwargs):
"""Create a graph for deconvolution, adding to the model
:param dirty_list:
:param psf_list:
:param model_imagelist:
:param prefix: Informative prefix to log messages
:param mask: Mask for deconvolution
:param kwargs: Parameters for functions in components
:return: graph for the deconvolution
"""
nchan = len(dirty_list)
# Number of moments. 1 is the sum.
nmoment = get_parameter(kwargs, "nmoment", 1)
if get_parameter(kwargs, "use_serial_clean", False):
from workflows.serial.imaging.imaging_serial import deconvolve_list_serial_workflow
return arlexecute.execute(deconvolve_list_serial_workflow, nout=nchan) \
(dirty_list, psf_list, model_imagelist, prefix=prefix, mask=mask, **kwargs)
def deconvolve(dirty, psf, model, facet, gthreshold, msk=None):
if prefix == '':
lprefix = "facet %d" % facet
else:
lprefix = "%s, facet %d" % (prefix, facet)
if nmoment > 0:
moment0 = calculate_image_frequency_moments(dirty)
this_peak = numpy.max(numpy.abs(moment0.data[0, ...])) / dirty.data.shape[0]
else:
ref_chan = dirty.data.shape[0] // 2
this_peak = numpy.max(numpy.abs(dirty.data[ref_chan, ...]))
if this_peak > 1.1 * gthreshold:
kwargs['threshold'] = gthreshold
result, _ = deconvolve_cube(dirty, psf, prefix=lprefix, mask=msk, **kwargs)
if result.data.shape[0] == model.data.shape[0]:
result.data += model.data
return result
else:
return copy_image(model)
deconvolve_facets = get_parameter(kwargs, 'deconvolve_facets', 1)
deconvolve_overlap = get_parameter(kwargs, 'deconvolve_overlap', 0)
deconvolve_taper = get_parameter(kwargs, 'deconvolve_taper', None)
if deconvolve_facets > 1 and deconvolve_overlap > 0:
deconvolve_number_facets = (deconvolve_facets - 2) ** 2
else:
deconvolve_number_facets = deconvolve_facets ** 2
scattered_channels_facets_model_list = \
[arlexecute.execute(image_scatter_facets, nout=deconvolve_number_facets)(m, facets=deconvolve_facets,
overlap=deconvolve_overlap,
taper=deconvolve_taper)
for m in model_imagelist]
scattered_facets_model_list = [
arlexecute.execute(image_gather_channels, nout=1)([scattered_channels_facets_model_list[chan][facet]
for chan in range(nchan)])
for facet in range(deconvolve_number_facets)]
# Scatter the separate channel images into deconvolve facets and then gather channels for each facet.
# This avoids constructing the entire spectral cube.
# i.e. SCATTER BY FACET then SCATTER BY CHANNEL
dirty_list_trimmed = arlexecute.execute(remove_sumwt, nout=nchan)(dirty_list)
scattered_channels_facets_dirty_list = \
[arlexecute.execute(image_scatter_facets, nout=deconvolve_number_facets)(d, facets=deconvolve_facets,
overlap=deconvolve_overlap,
taper=deconvolve_taper)
for d in dirty_list_trimmed]
scattered_facets_dirty_list = [
arlexecute.execute(image_gather_channels, nout=1)([scattered_channels_facets_dirty_list[chan][facet]
for chan in range(nchan)])
for facet in range(deconvolve_number_facets)]
psf_list_trimmed = arlexecute.execute(remove_sumwt, nout=nchan)(psf_list)
def extract_psf(psf, facets):
spsf = create_empty_image_like(psf)
cx = spsf.shape[3] // 2
cy = spsf.shape[2] // 2
wx = spsf.shape[3] // facets
wy = spsf.shape[2] // facets
xbeg = cx - wx // 2
xend = cx + wx // 2
ybeg = cy - wy // 2
yend = cy + wy // 2
spsf.data = psf.data[..., ybeg:yend, xbeg:xend]
spsf.wcs.wcs.crpix[0] -= xbeg
spsf.wcs.wcs.crpix[1] -= ybeg
return spsf
psf_list_trimmed = [arlexecute.execute(extract_psf)(p, deconvolve_facets) for p in psf_list_trimmed]
psf_centre = arlexecute.execute(image_gather_channels, nout=1)([psf_list_trimmed[chan]
for chan in range(nchan)])
# Work out the threshold. Need to find global peak over all dirty_list images
threshold = get_parameter(kwargs, "threshold", 0.0)
fractional_threshold = get_parameter(kwargs, "fractional_threshold", 0.1)
nmoment = get_parameter(kwargs, "nmoment", 1)
use_moment0 = nmoment > 0
# Find the global threshold. This uses the peak in the average on the frequency axis since we
# want to use it in a stopping criterion in a moment clean
global_threshold = arlexecute.execute(threshold_list, nout=1)(scattered_facets_dirty_list, threshold,
fractional_threshold,
use_moment0=use_moment0, prefix=prefix)
facet_list = numpy.arange(deconvolve_number_facets).astype('int')
if mask is None:
scattered_results_list = [
arlexecute.execute(deconvolve, nout=1)(d, psf_centre, m, facet,
global_threshold)
for d, m, facet in zip(scattered_facets_dirty_list, scattered_facets_model_list, facet_list)]
else:
mask_list = \
arlexecute.execute(image_scatter_facets, nout=deconvolve_number_facets)(mask,
facets=deconvolve_facets,
overlap=deconvolve_overlap)
scattered_results_list = [
arlexecute.execute(deconvolve, nout=1)(d, psf_centre, m, facet,
global_threshold, msk)
for d, m, facet, msk in
zip(scattered_facets_dirty_list, scattered_facets_model_list, facet_list, mask_list)]
# We want to avoid constructing the entire cube so we do the inverse of how we got here:
# i.e. SCATTER BY CHANNEL then GATHER BY FACET
# Gather the results back into one image, correcting for overlaps as necessary. The taper function is is used to
# feather the facets together
# gathered_results_list = arlexecute.execute(image_gather_facets, nout=1)(scattered_results_list,
# deconvolve_model_imagelist,
# facets=deconvolve_facets,
# overlap=deconvolve_overlap,
# taper=deconvolve_taper)
# result_list = arlexecute.execute(image_scatter_channels, nout=nchan)(gathered_results_list, subimages=nchan)
scattered_channel_results_list = [arlexecute.execute(image_scatter_channels, nout=nchan)(scat, subimages=nchan)
for scat in scattered_results_list]
# The structure is now [[channels] for facets]. We do the reverse transpose to the one above.
result_list = [arlexecute.execute(image_gather_facets, nout=1)([scattered_channel_results_list[facet][chan]
for facet in range(deconvolve_number_facets)],
model_imagelist[chan], facets=deconvolve_facets,
overlap=deconvolve_overlap)
for chan in range(nchan)]
return arlexecute.optimize(result_list)
def predict_skymodel_list_arlexecute_workflow(obsvis,
skymodel_list,
context,
vis_slices=1,
facets=1,
gcfcf=None,
docal=False,
**kwargs):
"""Predict from a list of skymodels, producing one visibility per skymodel
:param obsvis: "Observed Visibility"
:param skymodel_list: skymodel list
:param vis_slices: Number of vis slices (w stack or timeslice)
:param facets: Number of facets (per axis)
:param context: Type of processing e.g. 2d, wstack, timeslice or facets
:param gcfcg: tuple containing grid correction and convolution function
:param docal: Apply calibration table in skymodel
:param kwargs: Parameters for functions in components
:return: List of vis_lists
"""
def ft_cal_sm(ov, sm, g):
assert isinstance(ov, Visibility), ov
assert isinstance(sm, SkyModel), sm
if g is not None:
assert len(g) == 2, g
assert isinstance(g[0], Image), g[0]
assert isinstance(g[1], ConvolutionFunction), g[1]
v = copy_visibility(ov)
v.data['vis'][...] = 0.0 + 0.0j
if len(sm.components) > 0:
if isinstance(sm.mask, Image):
comps = copy_skycomponent(sm.components)
comps = apply_beam_to_skycomponent(comps, sm.mask)
v = predict_skycomponent_visibility(v, comps)
else:
v = predict_skycomponent_visibility(v, sm.components)
if isinstance(sm.image, Image):
if numpy.max(numpy.abs(sm.image.data)) > 0.0:
if isinstance(sm.mask, Image):
model = copy_image(sm.image)
model.data *= sm.mask.data
else:
model = sm.image
v = predict_list_serial_workflow([v], [model],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=[g],
**kwargs)[0]
if docal and isinstance(sm.gaintable, GainTable):
bv = convert_visibility_to_blockvisibility(v)
bv = apply_gaintable(bv, sm.gaintable, inverse=True)
v = convert_blockvisibility_to_visibility(bv)
return v
if gcfcf is None:
return [
arlexecute.execute(ft_cal_sm, nout=1)(obsvis, sm, None)
for ism, sm in enumerate(skymodel_list)
]
else:
return [
arlexecute.execute(ft_cal_sm, nout=1)(obsvis, sm, gcfcf[ism])
for ism, sm in enumerate(skymodel_list)
]
def mpccal_skymodel_list_arlexecute_workflow(visobs,
model,
theta_list,
nmajor=10,
context='2d',
mpccal_progress=None,
**kwargs):
"""Run MPC pipeline
This runs the Model Partition Calibration algorithm. See SDP Memo 97 for more details,
and see workflows/scripts/pipelines/mpccal_arlexecute_pipeline.py for an example of the application
:param visobs: Visibility (not a list!)
:param model: Model image
:param theta_list: List of SkyModels i.e. theta in memo 97.
:param nmajor: Number of major cycles
:param context: Imaging context
:param mpccal_progress: Function to display progress
:return: Delayed tuple (theta_list, residual)
"""
psf_obs = invert_list_arlexecute_workflow([visobs], [model],
context=context,
dopsf=True)
result = arlexecute.execute((theta_list, model))
for iteration in range(nmajor):
# The E step of decoupling the data models
vdatamodel_list = predict_skymodel_list_arlexecute_workflow(
visobs, theta_list, context=context, docal=True, **kwargs)
vdatamodel_list = crosssubtract_datamodels_skymodel_list_arlexecute_workflow(
visobs, vdatamodel_list)
# The M step: 1 - Update the models by deconvolving the residual image. The residual image must be calculated
# from a difference of the dirty images from the data model, and the dirty images
dirty_all_conv = convolve_skymodel_list_arlexecute_workflow(
visobs, theta_list, context=context, docal=True, **kwargs)
dirty_all_cal = invert_skymodel_list_arlexecute_workflow(
vdatamodel_list, theta_list, context=context, docal=True, **kwargs)
def diff_dirty(dcal, dconv):
assert numpy.max(numpy.abs(
dcal[0].data)) > 0.0, "before: dcal subimage is zero"
dcal[0].data -= dconv[0].data
assert numpy.max(numpy.abs(
dcal[0].data)) > 0.0, "after: dcal subimage is zero"
return dcal
dirty_all_cal = [
arlexecute.execute(diff_dirty, nout=1)(dirty_all_cal[i],
dirty_all_conv[i])
for i in range(len(dirty_all_cal))
]
def make_residual(dcal, tl, it):
res = create_empty_image_like(dcal[0][0])
for i, d in enumerate(dcal):
assert numpy.max(numpy.abs(
d[0].data)) > 0.0, "Residual subimage is zero"
if tl[i].mask is None:
res.data += d[0].data
else:
assert numpy.max(numpy.abs(
tl[i].mask.data)) > 0.0, "Mask image is zero"
res.data += d[0].data * tl[i].mask.data
assert numpy.max(numpy.abs(
res.data)) > 0.0, "Residual image is zero"
# import matplotlib.pyplot as plt
# from processing_components.image.operations import show_image
# show_image(res, title='MPCCAL residual image, iteration %d' % it)
# plt.show()
return res
residual = arlexecute.execute(make_residual,
nout=1)(dirty_all_cal, theta_list,
iteration)
deconvolved = deconvolve_list_arlexecute_workflow([(residual, 1.0)],
[psf_obs[0]],
[model], **kwargs)
# The M step: 2 - Update the gaintables
vpredicted_list = predict_skymodel_list_arlexecute_workflow(
visobs, theta_list, context=context, docal=True, **kwargs)
vcalibrated_list, gaintable_list = calibrate_list_arlexecute_workflow(
vdatamodel_list,
vpredicted_list,
calibration_context='T',
iteration=iteration,
global_solution=False,
**kwargs)
if mpccal_progress is not None:
theta_list = arlexecute.execute(mpccal_progress,
nout=len(theta_list))(
residual, theta_list,
gaintable_list, iteration)
theta_list = \
arlexecute.execute(update_skymodel_from_image, nout=len(theta_list))(theta_list, deconvolved[0])
theta_list = arlexecute.execute(update_skymodel_from_gaintables,
nout=len(theta_list))(
theta_list,
gaintable_list,
calibration_context='T')
result = arlexecute.execute((theta_list, residual))
return result
def invert_list_arlexecute_workflow(vis_list,
template_model_imagelist,
context,
dopsf=False,
normalize=True,
facets=1,
vis_slices=1,
gcfcf=None,
**kwargs):
""" Sum results from invert, iterating over the scattered image and vis_list
:param vis_list:
:param template_model_imagelist: Model used to determine image parameters
:param dopsf: Make the PSF instead of the dirty image
:param facets: Number of facets
:param normalize: Normalize by sumwt
:param vis_slices: Number of slices
:param context: Imaging context
:param gcfcg: tuple containing grid correction and convolution function
:param kwargs: Parameters for functions in components
:return: List of (image, sumwt) tuple
"""
if get_parameter(kwargs, "use_serial_invert", False):
from workflows.serial.imaging.imaging_serial import invert_list_serial_workflow
return [arlexecute.execute(invert_list_serial_workflow, nout=1) \
(vis_list=[vis_list[i]], template_model_imagelist=[template_model_imagelist[i]],
context=context, dopsf=dopsf, normalize=normalize, vis_slices=vis_slices,
facets=facets, gcfcf=gcfcf, **kwargs)[0]
for i, _ in enumerate(vis_list)]
if not isinstance(template_model_imagelist, collections.Iterable):
template_model_imagelist = [template_model_imagelist]
c = imaging_context(context)
vis_iter = c['vis_iterator']
invert = c['invert']
if facets % 2 == 0 or facets == 1:
actual_number_facets = facets
else:
actual_number_facets = max(1, (facets - 1))
def gather_image_iteration_results(results, template_model):
result = create_empty_image_like(template_model)
i = 0
sumwt = numpy.zeros([template_model.nchan, template_model.npol])
for dpatch in image_scatter_facets(result, facets=facets):
assert i < len(
results), "Too few results in gather_image_iteration_results"
if results[i] is not None:
assert len(results[i]) == 2, results[i]
dpatch.data[...] = results[i][0].data[...]
sumwt += results[i][1]
i += 1
return result, sumwt
def invert_ignore_none(vis, model, g):
if vis is not None:
return invert(vis,
model,
context=context,
dopsf=dopsf,
normalize=normalize,
gcfcf=g,
**kwargs)
else:
return create_empty_image_like(model), 0.0
# If we are doing facets, we need to create the gcf for each image
if gcfcf is None and facets == 1:
gcfcf = [
arlexecute.execute(create_pswf_convolutionfunction)(
template_model_imagelist[0])
]
# Loop over all vis_lists independently
results_vislist = list()
if facets == 1:
for freqwin, vis_list in enumerate(vis_list):
if len(gcfcf) > 1:
g = gcfcf[freqwin]
else:
g = gcfcf[0]
# Create the graph to divide the visibility into slices. This is by copy.
sub_vis_lists = arlexecute.execute(visibility_scatter,
nout=vis_slices)(
vis_list,
vis_iter,
vis_slices=vis_slices)
# Iterate within each vis_list
vis_results = list()
for sub_vis_list in sub_vis_lists:
vis_results.append(
arlexecute.execute(invert_ignore_none, pure=True)(
sub_vis_list, template_model_imagelist[freqwin], g))
results_vislist.append(
arlexecute.execute(sum_invert_results)(vis_results))
return results_vislist
else:
for freqwin, vis_list in enumerate(vis_list):
# Create the graph to divide an image into facets. This is by reference.
facet_lists = arlexecute.execute(
image_scatter_facets, nout=actual_number_facets**2)(
template_model_imagelist[freqwin], facets=facets)
# Create the graph to divide the visibility into slices. This is by copy.
sub_vis_lists = arlexecute.execute(visibility_scatter,
nout=vis_slices)(
vis_list,
vis_iter,
vis_slices=vis_slices)
# Iterate within each vis_list
vis_results = list()
for sub_vis_list in sub_vis_lists:
facet_vis_results = list()
for facet_list in facet_lists:
facet_vis_results.append(
arlexecute.execute(invert_ignore_none,
pure=True)(sub_vis_list, facet_list,
None))
vis_results.append(
arlexecute.execute(gather_image_iteration_results, nout=1)(
facet_vis_results, template_model_imagelist[freqwin]))
results_vislist.append(
arlexecute.execute(sum_invert_results)(vis_results))
return results_vislist
arlexecute.set_client(use_dask=True,
threads_per_worker=1,
memory_limit=4e9,
local_dir=dask_dir)
print(arlexecute.client)
arlexecute.run(init_logging)
nfreqwin = 41
ntimes = 5
rmax = 750.0
centre = nfreqwin // 2
# Load data from previous simulation
vis_list = [
arlexecute.execute(import_blockvisibility_from_hdf5)(arl_path(
'%s/ska-pipeline_simulation_vislist_%d.hdf' % (results_dir, v)))
for v in range(nfreqwin)
]
print('Reading visibilities')
vis_list = arlexecute.persist(vis_list)
cellsize = 0.001
npixel = 1024
pol_frame = PolarisationFrame("stokesI")
model_list = [
arlexecute.execute(create_image_from_visibility)(
v, npixel=npixel, cellsize=cellsize, polarisation_frame=pol_frame)
for v in vis_list
]
def deconvolve_list_arlexecute_workflow(dirty_list,
psf_list,
model_imagelist,
prefix='',
**kwargs):
"""Create a graph for deconvolution, adding to the model
:param dirty_list:
:param psf_list:
:param model_imagelist:
:param kwargs: Parameters for functions in components
:return: (graph for the deconvolution, graph for the flat)
"""
nchan = len(dirty_list)
nmoment = get_parameter(kwargs, "nmoment", 0)
def deconvolve(dirty, psf, model, facet, gthreshold):
if prefix == '':
lprefix = "facet %d" % facet
else:
lprefix = "%s, facet %d" % (prefix, facet)
if nmoment > 0:
moment0 = calculate_image_frequency_moments(dirty)
this_peak = numpy.max(numpy.abs(
moment0.data[0, ...])) / dirty.data.shape[0]
else:
ref_chan = dirty.data.shape[0] // 2
this_peak = numpy.max(numpy.abs(dirty.data[ref_chan, ...]))
if this_peak > 1.1 * gthreshold:
kwargs['threshold'] = gthreshold
result, _ = deconvolve_cube(dirty, psf, prefix=lprefix, **kwargs)
if result.data.shape[0] == model.data.shape[0]:
result.data += model.data
return result
else:
return copy_image(model)
deconvolve_facets = get_parameter(kwargs, 'deconvolve_facets', 1)
deconvolve_overlap = get_parameter(kwargs, 'deconvolve_overlap', 0)
deconvolve_taper = get_parameter(kwargs, 'deconvolve_taper', None)
if deconvolve_overlap > 0:
deconvolve_number_facets = (deconvolve_facets - 2)**2
else:
deconvolve_number_facets = deconvolve_facets**2
model_imagelist = arlexecute.execute(image_gather_channels,
nout=1)(model_imagelist)
# Scatter the separate channel images into deconvolve facets and then gather channels for each facet.
# This avoids constructing the entire spectral cube.
dirty_list_trimmed = arlexecute.execute(remove_sumwt,
nout=nchan)(dirty_list)
scattered_channels_facets_dirty_list = \
[arlexecute.execute(image_scatter_facets, nout=deconvolve_number_facets)(d, facets=deconvolve_facets,
overlap=deconvolve_overlap,
taper=deconvolve_taper)
for d in dirty_list_trimmed]
# Now we do a transpose and gather
scattered_facets_list = [
arlexecute.execute(image_gather_channels, nout=1)([
scattered_channels_facets_dirty_list[chan][facet]
for chan in range(nchan)
]) for facet in range(deconvolve_number_facets)
]
psf_list_trimmed = arlexecute.execute(remove_sumwt, nout=nchan)(psf_list)
psf_list_trimmed = arlexecute.execute(image_gather_channels,
nout=1)(psf_list_trimmed)
scattered_model_imagelist = \
arlexecute.execute(image_scatter_facets, nout=deconvolve_number_facets)(model_imagelist,
facets=deconvolve_facets,
overlap=deconvolve_overlap)
# Work out the threshold. Need to find global peak over all dirty_list images
threshold = get_parameter(kwargs, "threshold", 0.0)
fractional_threshold = get_parameter(kwargs, "fractional_threshold", 0.1)
nmoment = get_parameter(kwargs, "nmoment", 0)
use_moment0 = nmoment > 0
# Find the global threshold. This uses the peak in the average on the frequency axis since we
# want to use it in a stopping criterion in a moment clean
global_threshold = arlexecute.execute(threshold_list,
nout=1)(scattered_facets_list,
threshold,
fractional_threshold,
use_moment0=use_moment0,
prefix=prefix)
facet_list = numpy.arange(deconvolve_number_facets).astype('int')
scattered_results_list = [
arlexecute.execute(deconvolve, nout=1)(d, psf_list_trimmed, m, facet,
global_threshold)
for d, m, facet in zip(scattered_facets_list,
scattered_model_imagelist, facet_list)
]
# Gather the results back into one image, correcting for overlaps as necessary. The taper function is is used to
# feather the facets together
gathered_results_list = arlexecute.execute(image_gather_facets, nout=1)(
scattered_results_list,
model_imagelist,
facets=deconvolve_facets,
overlap=deconvolve_overlap,
taper=deconvolve_taper)
flat_list = arlexecute.execute(image_gather_facets,
nout=1)(scattered_results_list,
model_imagelist,
facets=deconvolve_facets,
overlap=deconvolve_overlap,
taper=deconvolve_taper,
return_flat=True)
return arlexecute.execute(image_scatter_channels,
nout=nchan)(gathered_results_list,
subimages=nchan), flat_list
input_vis = [arl_path('data/vis/sim-1.ms'), arl_path('data/vis/sim-2.ms')]
import time
start = time.time()
def load_ms(c):
v1 = create_visibility_from_ms(input_vis[0], channum=[c])[0]
v2 = create_visibility_from_ms(input_vis[1], channum=[c])[0]
vf = append_visibility(v1, v2)
vf = convert_visibility_to_stokes(vf)
vf.configuration.diameter[...] = 35.0
rows = vis_select_uvrange(vf, 0.0, uvmax=uvmax)
return create_visibility_from_rows(vf, rows)
vis_list = [arlexecute.execute(load_ms)(c) for c in range(nchan)]
print('Reading visibilities')
vis_list = arlexecute.persist(vis_list)
pol_frame = PolarisationFrame("stokesIQUV")
model_list = [
arlexecute.execute(create_image_from_visibility)(
v, npixel=npixel, cellsize=cellsize, polarisation_frame=pol_frame)
for v in vis_list
]
model_list = arlexecute.persist(model_list)
if args.serial_invert == 'True':
def actualSetUp(self,
add_errors=False,
freqwin=7,
block=False,
dospectral=True,
dopol=False,
zerow=True):
self.npixel = 256
self.low = create_named_configuration('LOWBD2', rmax=750.0)
self.freqwin = freqwin
self.vis_list = list()
self.ntimes = 5
cellsize = 0.001
self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
if freqwin > 1:
self.channelwidth = numpy.array(
freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.channelwidth = numpy.array([1e6])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
if dospectral:
flux = numpy.array(
[f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency])
else:
flux = numpy.array([f])
self.phasecentre = SkyCoord(ra=+180.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
self.vis_list = [
arlexecute.execute(ingest_unittest_visibility)(
self.low, [self.frequency[freqwin]],
[self.channelwidth[freqwin]],
self.times,
self.vis_pol,
self.phasecentre,
block=block,
zerow=zerow) for freqwin, _ in enumerate(self.frequency)
]
self.model_imagelist = [
arlexecute.execute(create_unittest_model,
nout=freqwin)(self.vis_list[freqwin],
self.image_pol,
cellsize=cellsize,
npixel=self.npixel)
for freqwin, _ in enumerate(self.frequency)
]
self.componentlist = [
arlexecute.execute(create_unittest_components)(
self.model_imagelist[freqwin],
flux[freqwin, :][numpy.newaxis, :])
for freqwin, _ in enumerate(self.frequency)
]
self.model_imagelist = [
arlexecute.execute(insert_skycomponent,
nout=1)(self.model_imagelist[freqwin],
self.componentlist[freqwin])
for freqwin, _ in enumerate(self.frequency)
]
self.vis_list = [
arlexecute.execute(predict_skycomponent_visibility)(
self.vis_list[freqwin], self.componentlist[freqwin])
for freqwin, _ in enumerate(self.frequency)
]
# Calculate the model convolved with a Gaussian.
self.model_imagelist = arlexecute.compute(self.model_imagelist,
sync=True)
model = self.model_imagelist[0]
self.cmodel = smooth_image(model)
export_image_to_fits(
model,
'%s/test_imaging_arlexecute_deconvolved_model.fits' % self.dir)
export_image_to_fits(
self.cmodel,
'%s/test_imaging_arlexecute_deconvolved_cmodel.fits' % self.dir)
if add_errors and block:
self.vis_list = [
arlexecute.execute(insert_unittest_errors)(self.vis_list[i])
for i, _ in enumerate(self.frequency)
]
# self.vis_list = arlexecute.compute(self.vis_list, sync=True)
self.vis_list = arlexecute.persist(self.vis_list)
self.model_imagelist = arlexecute.scatter(self.model_imagelist)
def simulation(self,
args,
time_series='wind',
band='B2',
context='singlesource',
vp_directory=''):
ra = args.ra
declination = args.declination
use_radec = args.use_radec == "True"
integration_time = args.integration_time
time_range = args.time_range
time_chunk = args.time_chunk
offset_dir = args.offset_dir
pbtype = args.pbtype
pbradius = args.pbradius
rmax = args.rmax
flux_limit = args.flux_limit
npixel = args.npixel
shared_directory = args.shared_directory
vp_directory = args.vp_directory
# Simulation specific parameters
global_pe = numpy.array(args.global_pe)
static_pe = numpy.array(args.static_pe)
dynamic_pe = args.dynamic_pe
seed = args.seed
basename = os.path.basename(os.getcwd())
# client = get_dask_Client()
use_dask = False
arlexecute.set_client(use_dask=use_dask)
# Set up details of simulated observation
nfreqwin = 1
if band == 'B1':
frequency = [0.765e9]
elif band == 'B2':
frequency = [1.36e9]
elif band == 'Ku':
frequency = [12.179e9]
else:
raise ValueError("Unknown band %s" % band)
phasecentre = SkyCoord(ra=ra * u.deg,
dec=declination * u.deg,
frame='icrs',
equinox='J2000')
bvis_graph = create_standard_mid_simulation_arlexecute_workflow(
band, rmax, phasecentre, time_range, time_chunk, integration_time,
shared_directory)
future_bvis_list = arlexecute.persist(bvis_graph)
vis_graph = [
arlexecute.execute(convert_blockvisibility_to_visibility)(bv)
for bv in future_bvis_list
]
future_vis_list = arlexecute.persist(vis_graph, sync=True)
# We need the HWHM of the primary beam, and the location of the nulls
HWHM_deg, null_az_deg, null_el_deg = find_pb_width_null(
pbtype, frequency)
HWHM = HWHM_deg * numpy.pi / 180.0
FOV_deg = 8.0 * 1.36e9 / frequency[0]
advice_list = arlexecute.execute(advise_wide_field)(
future_vis_list[0], guard_band_image=1.0, delA=0.02, verbose=False)
advice = arlexecute.compute(advice_list, sync=True)
pb_npixel = 1024
d2r = numpy.pi / 180.0
pb_cellsize = d2r * FOV_deg / pb_npixel
cellsize = advice['cellsize']
# Now construct the components
original_components, offset_direction = create_simulation_components(
context, phasecentre, frequency, pbtype, offset_dir, flux_limit,
pbradius * HWHM, pb_npixel, pb_cellsize)
vp_list = [
arlexecute.execute(create_image_from_visibility)(
bv,
npixel=pb_npixel,
frequency=frequency,
nchan=nfreqwin,
cellsize=pb_cellsize,
phasecentre=phasecentre,
override_cellsize=False) for bv in future_bvis_list
]
vp_list = [
arlexecute.execute(create_vp)(vp,
pbtype,
pointingcentre=phasecentre,
use_local=not use_radec)
for vp in vp_list
]
future_vp_list = arlexecute.persist(vp_list)
# Make one image per component
future_model_list = [
arlexecute.execute(create_image_from_visibility)(
future_vis_list[0],
npixel=npixel,
frequency=frequency,
nchan=nfreqwin,
cellsize=cellsize,
phasecentre=offset_direction,
polarisation_frame=PolarisationFrame("stokesI"))
for i, _ in enumerate(original_components)
]
a2r = numpy.pi / (3600.0 * 1800)
no_error_gtl = None
error_gtl = None
if time_series == '':
global_pointing_error = global_pe
static_pointing_error = static_pe
pointing_error = dynamic_pe
no_error_gtl, error_gtl = \
create_pointing_errors_gaintable_arlexecute_workflow(future_bvis_list, original_components,
sub_vp_list=future_vp_list,
use_radec=use_radec,
pointing_error=a2r * pointing_error,
static_pointing_error=a2r * static_pointing_error,
global_pointing_error=a2r * global_pointing_error,
seed=seed,
show=False, basename=basename)
elif time_series == 'wind':
no_error_gtl, error_gtl = \
create_pointing_errors_gaintable_arlexecute_workflow(future_bvis_list, original_components,
sub_vp_list=future_vp_list,
use_radec=use_radec,
time_series=time_series,
time_series_type='precision',
seed=seed,
show=False, basename=basename)
elif time_series == 'gravity':
no_error_gtl, error_gtl = \
create_surface_errors_gaintable_arlexecute_workflow(band, future_bvis_list, original_components,
vp_directory=vp_directory, use_radec=use_radec,
show=False, basename=basename)
else:
raise ValueError("Unknown type of error %s" % time_series)
# Now make all the residual images
vis_comp_chunk_dirty_list = \
calculate_residual_from_gaintables_arlexecute_workflow(future_bvis_list, original_components,
future_model_list,
no_error_gtl, error_gtl)
# Add the resulting images
error_dirty_list = sum_invert_results_arlexecute(
vis_comp_chunk_dirty_list)
# Actually compute the graph assembled above
error_dirty, sumwt = arlexecute.compute(error_dirty_list, sync=True)
return error_dirty, sumwt
