# CubiCal: a radio interferometric calibration suite
# (c) 2017 Rhodes University & Jonathan S. Kenyon
# http://github.com/ratt-ru/CubiCal
# This code is distributed under the terms of GPLv2, see LICENSE.md for details
"""
Handles the flagging of data.
"""
# This is to keep matplotlib from falling over when no DISPLAY is set (which it otherwise does,
# even if one is only trying to save figures to .png.
import numpy as np
import re
from cubical.tools import logger, ModColor
import cubical.plots as plots
log = logger.getLogger("flagging")
from collections import OrderedDict
class FL(object):
""" Namespace for flag bits. """
dtype = np.uint16 # dtype used for flag arrays
PRIOR = dtype(1 << 0) # prior flags (i.e. from MS)
MISSING = dtype(1 << 1) # missing data or solution
INVALID = dtype(1 << 2) # invalid data or model (inf, nan)
ILLCOND = dtype(1 << 3) # solution ill conditioned - bad inverse
NOCONV = dtype(1 << 4) # no convergence
CHISQ = dtype(1 << 5) # excessive chisq
GOOB = dtype(1 << 6) # gain solution out of bounds
# CubiCal: a radio interferometric calibration suite
# (c) 2017 Rhodes University & Jonathan S. Kenyon
# http://github.com/ratt-ru/CubiCal
# This code is distributed under the terms of GPLv2, see LICENSE.md for details
from __future__ import print_function
from cubical.machines.complex_2x2_machine import Complex2x2Gains
import numpy as np
import numpy.ma as ma
from numpy.ma import masked_array
from cubical.flagging import FL
import cubical.kernels
from cubical.tools import logger
log = logger.getLogger("complex_pol")
class PolarizationGains(Complex2x2Gains):
"""
This class implements the full complex 2x2 gain machine.
"""
def __init__(self, label, data_arr, ndir, nmod, chunk_ts, chunk_fs,
chunk_label, options):
"""
Initialises a 2x2 polarization gain machine (leakage and PZD)
Args:
label (str):
Label identifying the Jones term.
data_arr (np.ndarray):
Shape (n_mod, n_tim, n_fre, n_ant, n_ant, n_cor, n_cor) array containing observed
# CubiCal: a radio interferometric calibration suite
# (c) 2017 Rhodes University & Jonathan S. Kenyon
# http://github.com/ratt-ru/CubiCal
# This code is distributed under the terms of GPLv2, see LICENSE.md for details
from builtins import range
from cubical.database.pickled_db import PickledDatabase
from cubical.data_handler.ms_data_handler import MSDataHandler
from cubical.tools import logger
from pyrap.tables import table as tbl
import os
import shutil
import numpy as np
import subprocess
import six
log = logger.getLogger("casa_db_adaptor")
# to the tune of KATDAL :)
BLANK_TABLE_NAME = 'blankcaltable.CASA'
BLANK_TABLE_TARBALL = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"{}.tgz".format(BLANK_TABLE_NAME))
class casa_caltable_factory(object):
"""
Gaintable Factory
Useful methods for creating and writing CASA-style gaintables
"""
@classmethod
def init_empty(cls,
from __future__ import print_function
from builtins import range
import math, cmath
import numpy as np
import numpy.ma as ma
from cubical.tools import logger
log = logger.getLogger("plots")
from past.builtins import cmp
from cubical.plots import DPI, ZOOM, make_antenna_xaxis
def _abs(x):
"""
Works around numpy bug with abs() of masked arrays producing a
ComplexWarning(Casting complex values to real discards the imaginary part)
"""
if ma.isMA(x):
return ma.masked_array(np.abs(x.data), x.mask)
else:
return ma.masked_array(np.abs(x))
def _cmp_antenna(sa, sb):
"""Helper function to sort antenna names. Try numeric compare first, fall back to text compare if failed"""
try:
return cmp(int(sa), int(sb))
except:
return cmp(sa, sb)
from __future__ import print_function
from builtins import range
import numpy as np
import traceback
from cubical.tools import BREAK # useful: can set static breakpoints by putting BREAK() in the code
from cubical.tools import logger
log = logger.getLogger("madmax")
def make_dual_absres_plot(absres,
fl_prior,
fl_new,
p,
q,
metadata,
subplot_titles={}):
import pylab
feeds = metadata.feeds
# inv: data that was flagged prior to this mad max step
fl_prior = fl_prior[:, :, p, q] & ~fl_new[:, :, p, q]
figure = pylab.figure(figsize=(16, 10))
resmask = np.zeros_like(absres[0, :, :, p, q], dtype=bool)
resmask[:] = fl_prior[..., np.newaxis,
np.newaxis] #| absres[0, :, :, p, q]==0
res = np.ma.masked_array(absres[0, :, :, p, q], resmask)
resmin, resmax = res.min(), res.max()
vmin = float(max(resmin if resmin is not np.ma.masked else 0, 1e-9))
vmax = float(resmax if resmax is not np.ma.masked else 1)
if vmin != vmax:
"""
from __future__ import print_function
from builtins import range
import numpy as np
import montblanc
import montblanc.util as mbu
import montblanc.impl.rime.tensorflow.ms.ms_manager as MS
from montblanc.impl.rime.tensorflow.sources import SourceProvider
from montblanc.impl.rime.tensorflow.sinks import SinkProvider
import datetime as dt
import pyrap.quanta as pq
from cubical.tools import logger, ModColor
log = logger.getLogger("MBSourceProvider")
class MSSourceProvider(SourceProvider):
"""
Handles interface between CubiCal tiles and Montblanc simulation.
"""
def __init__(self,
tile,
time_col,
antea,
anteb,
ddid_col,
uvw,
freqs,
sort_ind,
# http://github.com/ratt-ru/CubiCal
# This code is distributed under the terms of GPLv2, see LICENSE.md for details
"""
Handles solver statistics.
"""
from __future__ import print_function
from builtins import range
import math
import numpy as np
from future.moves import pickle
from cubical.tools import logger
from cubical.tools import ModColor
from collections import OrderedDict
log = logger.getLogger("stats")
class SolverStats(object):
"""
SolverStats is a container for various per-chunk statistics collected during the solving
process.
"""
def __init__(self, obj):
"""
Initialisation for the SolverStats object.
Args:
obj (dict or np.ndarray or file):
Object from which to initialise the stats object.
# CubiCal: a radio interferometric calibration suite
# (c) 2017 Rhodes University & Jonathan S. Kenyon
# http://github.com/ratt-ru/CubiCal
# This code is distributed under the terms of GPLv2, see LICENSE.md for details
from __future__ import print_function
from cubical.machines.interval_gain_machine import PerIntervalGains
import numpy as np
from scipy import special
from cubical.flagging import FL
import cubical.kernels
import time
from cubical.tools import logger
log = logger.getLogger("robust_2x2") #TODO check this "complex_2x2"
class ComplexW2x2Gains(PerIntervalGains):
"""
This class implements the weighted full complex 2x2 gain machine based on the Complex T-distribution
"""
def __init__(self, label, data_arr, ndir, nmod,
chunk_ts, chunk_fs, chunk_label, options):
"""
Initialises a weighted complex 2x2 gain machine.
Args:
label (str):
Label identifying the Jones term.
data_arr (np.ndarray):
Shape (n_mod, n_tim, n_fre, n_ant, n_ant, n_cor, n_cor) array containing observed
from __future__ import print_function
from builtins import range
import pyrap.quanta as pq
import pyrap.measures
pm = pyrap.measures.measures()
import copy
import numpy as np
import datetime as dt
from cubical.tools import logger, ModColor
log = logger.getLogger("parallactic_machine")
class parallactic_machine(object):
def __init__(self,
observer_names,
ECEF_positions,
feed_angles=None,
epoch='J2000',
feed_basis='linear',
enable_rotation=True,
enable_pa=True,
enable_derotation=True,
field_centre=(0, -90)):
"""
Machine to rotate and derotate visibilities around 3rd axis of observer's local
coordiante frame
Args:
@observer_names: list of antenna names
@ECEF_positions: (nant, 3) array, as provided in MS::ANTENNA subtable
@feed_angles: (nant, 2) array, as provided by MS::FEED subtable. None for nominally oriented receptors
from __future__ import print_function
from psutil import virtual_memory
from cubical import workers
from cubical.tools import logger, ModColor
log = logger.getLogger("wisdom")
def estimate_mem(data_handler, tile_list, data_opts, dist_opts):
# Grab current memory information.
mem = virtual_memory()
tot_sys_mem = mem.total / (1024**3) # Convert to value in GiB.
# Grab an example tile - for now we assume that it is a representative
# example. In principle we could loop over all tiles and look for the
# absolute peak memeory usage. We do some fiddling to get the number of
# chunks along each axis.
example_tile = tile_list[0]
chunks_per_tile = example_tile.total_tf_chunks()
along_freq = len(data_handler.freqchunks[0])
along_time = chunks_per_tile // along_freq
# Figure out the chunk dimensions. The ors handle the 0 case. TODO: The
# time behaviour might not be safe.
t_per_chunk = example_tile.rowchunks[0].timeslice.stop
# CubiCal: a radio interferometric calibration suite
# (c) 2017 Rhodes University & Jonathan S. Kenyon
# http://github.com/ratt-ru/CubiCal
# This code is distributed under the terms of GPLv2, see LICENSE.md for details
from cubical.machines.interval_gain_machine import PerIntervalGains
import numpy as np
from cubical.flagging import FL
import cubical.kernels
from cubical.tools import logger
log = logger.getLogger("complex_2x2")
class Complex2x2Gains(PerIntervalGains):
"""
This class implements the full complex 2x2 gain machine.
"""
def __init__(self, label, data_arr, ndir, nmod, chunk_ts, chunk_fs,
chunk_label, options):
"""
Initialises a 2x2 complex gain machine.
Args:
label (str):
Label identifying the Jones term.
data_arr (np.ndarray):
Shape (n_mod, n_tim, n_fre, n_ant, n_ant, n_cor, n_cor) array containing observed
visibilities.
ndir (int):
Number of directions.
nmod (nmod):
# CubiCal: a radio interferometric calibration suite
# (c) 2017 Rhodes University & Jonathan S. Kenyon
# http://github.com/ratt-ru/CubiCal
# This code is distributed under the terms of GPLv2, see LICENSE.md for details
from __future__ import print_function
from cubical.machines.parameterised_machine import ParameterisedGains
import numpy as np
from numpy.ma import masked_array
import cubical.kernels
from cubical.tools import logger
log = logger.getLogger("slopes")
def _normalize(x, dtype):
"""
Helper function: normalizes array to [0,1] interval.
"""
if len(x) > 1:
return ((x - x[0]) / (x[-1] - x[0])).astype(dtype)
elif len(x) == 1:
return np.zeros(1, dtype)
else:
return x
import builtins
try:
builtins.profile
except AttributeError:
This file has been ported to CubiCal by the original authors, with minor modifications.
'''
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from DDFacet.compatibility import range
import numpy
import os
import os.path
import sys
from cubical.tools import logger, ModColor
log = logger.getLogger("FITSBeamInterpolator")
import pyrap.tables
import numpy as np
dm = pyrap.measures.measures()
dq = pyrap.quanta
# This a list of the Stokes enums (as defined in casacore header measures/Stokes.h)
# These are referenced by the CORR_TYPE column of the MS POLARIZATION subtable.
# E.g. 5,6,7,8 corresponds to RR,RL,LR,LL
MS_STOKES_ENUMS = [
"Undefined", "I", "Q", "U", "V", "RR", "RL", "LR", "LL", "XX", "XY", "YX", "YY", "RX", "RY", "LX", "LY", "XR", "XL", "YR", "YL", "PP", "PQ", "QP", "QQ", "RCircular", "LCircular", "Linear", "Ptotal", "Plinear", "PFtotal", "PFlinear", "Pangle"
];
import gc
import traceback
from cubical.tools import logger, ModColor
from cubical.flagging import FL
from cubical.statistics import SolverStats
from cubical.tools import BREAK # useful: can set static breakpoints by putting BREAK() in the code
## uncomment this to make UserWarnings (from e.g. numpy.ma) into full-blown exceptions
## TODO: add a --debug-catch-warnings option for this?
#import warnings
#warnings.simplefilter('error', UserWarning)
#warnings.simplefilter('error', RuntimeWarning)
from .madmax.flagger import Flagger
log = logger.getLogger("solver")
#log.verbosity(2)
# global defaults dict
GD = None
# MS metadata
metadata = None
# gain machine factory to use
gm_factory = None
# IFR-based gain machine to use
ifrgain_machine = None
# set to true for old-style (version <= 1.2.1) weight averaging, where 2x2 weights are collapsed into a single number
# CubiCal: a radio interferometric calibration suite
# (c) 2017 Rhodes University & Jonathan S. Kenyon
# http://github.com/ratt-ru/CubiCal
# This code is distributed under the terms of GPLv2, see LICENSE.md for details
from cubical.machines.interval_gain_machine import PerIntervalGains
import numpy as np
from scipy import special
from cubical.flagging import FL
import cubical.kernels
import time
from cubical.tools import logger
log = logger.getLogger("complex_2x2") #TODO check this
class ComplexW2x2Gains(PerIntervalGains):
"""
This class implements the weighted full complex 2x2 gain machine based on the Complex T-distribution
"""
def __init__(self, label, data_arr, ndir, nmod, chunk_ts, chunk_fs,
chunk_label, options):
"""
Initialises a weighted complex 2x2 gain machine.
Args:
label (str):
Label identifying the Jones term.
data_arr (np.ndarray):
Shape (n_mod, n_tim, n_fre, n_ant, n_ant, n_cor, n_cor) array containing observed
visibilities.
ndir (int):
except Exception as exc:
print("{}({})\n {}".format(type(exc).__name__, exc, traceback.format_exc()), file=log)
if name == name0:
print(ModColor.Str("Error substituting '{}', see above".format(name)), file=log)
else:
print(ModColor.Str("Error substituting '{}' (derived from '{}'), see above".format(name, name0)), file=log)
raise ValueError(name)
from cubical.data_handler.ms_data_handler import MSDataHandler
from cubical.tools import parsets, dynoptparse, shm_utils, ModColor
from cubical.machines import machine_types
from cubical.machines import jones_chain_machine, jones_chain_robust_machine
from cubical.machines import ifr_gain_machine
from cubical import workers
log = logger.getLogger("main")
import cubical.solver as solver
import cubical.flagging as flagging
from cubical.statistics import SolverStats
class UserInputError(Exception):
pass
# set to true with --Debug-Pdb 1, causes pdb to be invoked on exception
enable_pdb = True
def debug():
""" Calls the main() function in debugging mode. """
from DDFacet.Imager import ClassDDEGridMachine
if six.PY3:
import DDFacet.cbuild.Gridder._pyGridderSmearPolsClassic3x as _pyGridderSmearPolsClassic
else:
import DDFacet.cbuild.Gridder._pyGridderSmearPolsClassic27 as _pyGridderSmearPolsClassic
from DDFacet.ToolsDir.ModToolBox import EstimateNpix
from DDFacet.Array import shared_dict
from DDFacet.ToolsDir import ModFFTW
import DDFacet.ToolsDir.ModTaper as MT
except ImportError:
raise ImportError("Could not import DDFacet")
from . import FITSBeamInterpolator
from . import DicoSourceProvider
from cubical.tools import logger, ModColor
log = logger.getLogger("DDFacetSim")
import numpy as np
import math
import os
from concurrent.futures import (ProcessPoolExecutor as PPE,
ThreadPoolExecutor as TPE)
import hashlib
import datetime as dt
import pyrap.quanta as pq
from numba import jit, prange
import concurrent.futures as cf
import cubical.workers as cw
import multiprocessing
from astropy import wcs, coordinates, units
def init_degridders(dir_CFs,
# CubiCal: a radio interferometric calibration suite
# (c) 2017 Rhodes University & Jonathan S. Kenyon
# http://github.com/ratt-ru/CubiCal
# This code is distributed under the terms of GPLv2, see LICENSE.md for details
from __future__ import print_function
import numpy as np
from numpy.ma import masked_array
from cubical import param_db
from cubical.tools import logger, ModColor
log = logger.getLogger("gain_machine")
class IfrGainMachine(object):
"""
Interferometer-based gain machine.
Note that this class is outside of the normal gain machine hierarchy. It wraps all the
functionality required to manage IFR-based gain (aka baseline-based corrections)
calculations.
"""
def __init__(self, gmfactory, ifrgain_opts, compute=True):
"""
Initializes the IFR-based gains machinery.
Args:
gmfactory: a GainMachine Factory is used to manage the solution databases
ifrgain_opts: dict of options
compute: if False, gains are not computed even if options ask them to
# (c) 2017 Rhodes University & Jonathan S. Kenyon
# http://github.com/ratt-ru/CubiCal
# This code is distributed under the terms of GPLv2, see LICENSE.md for details
from __future__ import print_function
from builtins import range
from cubical.machines.abstract_machine import MasterMachine
from cubical.machines.complex_2x2_machine import Complex2x2Gains
from cubical.machines.complex_W_2x2_machine import ComplexW2x2Gains
import numpy as np
import cubical.kernels
from cubical.tools import logger
from . import machine_types
from cubical.flagging import FL
log = logger.getLogger("jones_chain")
class JonesChain(MasterMachine):
"""
This class implements a gain machine for an arbitrary chain of Jones matrices. Most of its
functionality is consistent with a complex 2x2 solver - many of its methods mimic those of the
underlying complex 2x2 machines.
"""
def __init__(self, label, data_arr, ndir, nmod, times, frequencies,
chunk_label, jones_options):
"""
Initialises a chain of complex 2x2 gain machines.
Args:
label (str):
# CubiCal: a radio interferometric calibration suite
# (c) 2017 Rhodes University & Jonathan S. Kenyon
# http://github.com/ratt-ru/CubiCal
# This code is distributed under the terms of GPLv2, see LICENSE.md for details
"""
Handles parameter databases which can contain solutions and other relevant values.
"""
import numpy as np
from cubical.tools import logger
log = logger.getLogger("param_db")
#from database.pickled_db import PickledDatabase
from database.casa_db_adaptor import casa_db_adaptor
def create(filename, metadata={}, backup=True):
"""
Creates a new parameter database.
Args:
filename (str):
Name of file to save DB to.
metadata (dict, optional):
Optional dictionary of metadata.
backup (bool, optional):
If True, and an old database with the same filename exists, make a backup.
Returns:
:obj:`~cubical.param_db.PickledDatabase`:
A resulting parameter database.
#!/usr/bin/env python
from __future__ import print_function
from cubical.tools import logger
from collections import OrderedDict
import matplotlib.patches as mpatches
from pylab import *
import re, fnmatch
import warnings
log = logger.getLogger("gain_plots")
class PlotLimits(object):
def __init__(self):
self.max_reim = None
self.max_reim_1 = None
self.max_phase = None
self.min_ampl = None
self.max_ampl = None
self.min_ampl_1 = None
self.max_ampl_1 = None
self.min_freq = None
self.max_freq = None
self.min_time = None
self.max_time = None
class PlotOptions(PlotLimits):
"""This holds a set of common plotting options"""
def __init__(self):
self.dpi = 150
