def compute_vis_noise(sefd):
"""Computes nominal per-visibility noise"""
sefd = sefd or SEFD
tab = ms.ms()
spwtab = ms.ms(subtable="SPECTRAL_WINDOW")
freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID, 0]
wavelength = 300e+6 / freq0
bw = spwtab.getcol("CHAN_WIDTH")[ms.SPWID, 0]
dt = tab.getcol("EXPOSURE", 0, 1)[0]
dtf = (tab.getcol("TIME",
tab.nrows() - 1, 1) - tab.getcol("TIME", 0, 1))[0]
# close tables properly, else the calls below will hang waiting for a lock...
tab.close()
spwtab.close()
info(
">>> $MS freq %.2f MHz (lambda=%.2fm), bandwidth %.2g kHz, %.2fs integrations, %.2fh synthesis"
% (freq0 * 1e-6, wavelength, bw * 1e-3, dt, dtf / 3600))
noise = sefd / math.sqrt(2 * bw * dt)
info(">>> SEFD of %.2f Jy gives per-visibility noise of %.2f mJy" %
(sefd, noise * 1000))
return noise
def compute_vis_noise (noise=0,sefd=SEFD):
"""Computes nominal per-visibility noise"""
tab = ms.ms();
spwtab = ms.ms(subtable="SPECTRAL_WINDOW");
global BASEFREQ
BASEFREQ = freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID,0];
global WAVELENGTH
WAVELENGTH = 300e+6/freq0
bw = BANDWIDTH or spwtab.getcol("CHAN_WIDTH")[ms.SPWID,0];
dt = INTEGRATION or tab.getcol("EXPOSURE",0,1)[0];
dtf = (tab.getcol("TIME",tab.nrows()-1,1)-tab.getcol("TIME",0,1))[0]
# close tables properly, else the calls below will hang waiting for a lock...
tab.close();
spwtab.close();
info(">>> $MS freq %.2f MHz (lambda=%.2fm), bandwidth %.2g kHz, %.2fs integrations, %.2fh synthesis"%(freq0*1e-6,WAVELENGTH,bw*1e-3,dt,dtf/3600));
_writestat("freq0",freq0);
_writestat("wavelength",WAVELENGTH);
_writestat("bw",bw);
_writestat("synthesis_time",dtf);
_writestat("integration",dt);
if not noise:
noise = sefd/math.sqrt(2*bw*dt);
info(">>> SEFD of %.2f Jy gives per-visibility noise of %.2f mJy"%(sefd,noise*1000));
else:
info(">>> using per-visibility noise of %.2f mJy"%(noise*1000));
_writestat("vis_noise",noise);
return noise;
def image_noise(sigma):
""" Estimates RMS in noise in naturally weighted image """
tab = ms.ms()
data = tab.getcol('DATA')
sigma_vis = sigma
N_vis = np.product(data.shape[:2])
return sigma_vis/math.sqrt(N_vis)
def image_noise(sigma):
""" Estimates RMS in noise in naturally weighted image """
tab = ms.ms()
data = tab.getcol('DATA')
sigma_vis = sigma
N_vis = np.product(data.shape[:2])
return sigma_vis / math.sqrt(N_vis)
def get_time_slots():
t = ms.ms()
A1 = t.getcol("ANTENNA1")
A2 = t.getcol("ANTENNA2")
uvw=t.getcol("UVW")
temp = uvw[(A1==0)&(A2==1),0]
return(len(temp))
def compute_vis_noise (noise=0):
tab = ms.ms().query("FIELD_ID==%d"%ms.FIELD);
spwtab = ms.ms(subtable="SPECTRAL_WINDOW");
freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID,0];
global WAVELENGTH
WAVELENGTH = 300e+6/freq0
bw = spwtab.getcol("CHAN_WIDTH")[ms.SPWID,0];
dt = INTEGRATION or tab.getcol("EXPOSURE",0,1)[0];
dtf = (tab.getcol("TIME",tab.nrows()-1,1)-tab.getcol("TIME",0,1))[0]
# close tables properly, else the calls below will hang waiting for a lock...
tab.close();
spwtab.close();
info(">>> $MS freq %.2f MHz (lambda=%.2fm), bandwidth %.2g kHz, %.2fs integrations, %.2fh synthesis"%(freq0*1e-6,WAVELENGTH,bw*1e-3,dt,dtf/3600));
if not noise:
noise = SEFD/math.sqrt(2*bw*dt);
info(">>> SEFD of %.2f Jy gives per-visibility noise of %.2f mJy"%(SEFD,noise*1000));
else:
info(">>> using per-visibility noise of %.2f mJy"%(noise*1000));
return noise;
def set_defaults(msname='$MS', psf=False):
""" Extract some basic information about the data"""
tab = ms.ms()
uvmax = max(tab.getcol("UVW")[:2].sum(0)**2)
res = 1.22 * ((2.998e8 / freq) / maxuv) / 2.
im.cellsize = "%farcsec" % (numpy.rad2deg(res / 6) * 3600)
im.stokes = 'I' # make a brightness map by default
im.npix = 2048
im.mode = 'channel'
im.IMAGE_CHANNELIZE = 0
def set_defaults(msname='$MS',psf=False):
""" Extract some basic information about the data"""
tab = ms.ms()
uvmax = max( tab.getcol("UVW")[:2].sum(0)**2 )
res = 1.22*((2.998e8/freq)/maxuv)/2.
im.cellsize = "%farcsec"%(numpy.rad2deg(res/6)*3600)
im.stokes = 'I' # make a brightness map by default
im.npix = 2048
im.mode = 'channel'
im.IMAGE_CHANNELIZE = 0
def flag_stepped_timeslot (step=3):
"""Flags every Nth timeslot"""
nant = ms.ms(subtable="ANTENNA").nrows();
tab = ms.msw();
nb = nant*(nant+1)/2
frow = tab.getcol("FLAG_ROW");
nr = len(frow);
nt = len(frow)/nb;
info("$MS has $nr rows, $nant antennas, $nb baselines and $nt timeslots, flagging every $step timeslots");
frow = frow.reshape([nt,nb]);
frow[::step,:] = True;
tab.putcol("FLAG_ROW",frow.reshape((nr,)));
def compute_vis_noise (sefd):
"""Computes nominal per-visibility noise"""
#sefd = sefd or SEFD
tab = ms.ms()
spwtab = ms.ms(subtable="SPECTRAL_WINDOW")
freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID,0]
wavelength = 300e+6/freq0
bw = spwtab.getcol("CHAN_WIDTH")[ms.SPWID,0]
dt = tab.getcol("EXPOSURE",0,1)[0]
dtf = (tab.getcol("TIME",tab.nrows()-1,1)-tab.getcol("TIME",0,1))[0]
# close tables properly, else the calls below will hang waiting for a lock...
tab.close()
spwtab.close()
info(">>> $MS freq %.2f MHz (lambda=%.2fm), bandwidth %.2g kHz, %.2fs integrations, %.2fh synthesis"%(freq0*1e-6,wavelength,bw*1e-3,dt,dtf/3600))
noise = sefd/math.sqrt(2*bw*dt)
info(">>> SEFD of %.2f Jy gives per-visibility noise of %.2f mJy"%(sefd,noise*1000))
return noise
def compute_vis_noise(noise=0):
tab = ms.ms().query("FIELD_ID==%d" % ms.FIELD)
spwtab = ms.ms(subtable="SPECTRAL_WINDOW")
freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID, 0]
global WAVELENGTH
WAVELENGTH = 300e+6 / freq0
bw = spwtab.getcol("CHAN_WIDTH")[ms.SPWID, 0]
dt = INTEGRATION or tab.getcol("EXPOSURE", 0, 1)[0]
dtf = (tab.getcol("TIME",
tab.nrows() - 1, 1) - tab.getcol("TIME", 0, 1))[0]
# close tables properly, else the calls below will hang waiting for a lock...
tab.close()
spwtab.close()
info(
">>> $MS freq %.2f MHz (lambda=%.2fm), bandwidth %.2g kHz, %.2fs integrations, %.2fh synthesis"
% (freq0 * 1e-6, WAVELENGTH, bw * 1e-3, dt, dtf / 3600))
if not noise:
noise = SEFD / math.sqrt(2 * bw * dt)
info(">>> SEFD of %.2f Jy gives per-visibility noise of %.2f mJy" %
(SEFD, noise * 1000))
else:
info(">>> using per-visibility noise of %.2f mJy" % (noise * 1000))
return noise
def simnoise (noise=0,rowchunk=100000,skipnoise=False,addToCol=None,scale_noise=1.0,column='MODEL_DATA'):
conf = MS.split('_')[0]
spwtab = ms.ms(subtable="SPECTRAL_WINDOW")
freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID,0]/1e6
tab = ms.msw()
dshape = list(tab.getcol('DATA').shape)
nrows = dshape[0]
noise = noise or compute_vis_noise()
if addToCol: colData = tab.getcol(addToCol)
for row0 in range(0,nrows,rowchunk):
nr = min(rowchunk,nrows-row0)
dshape[0] = nr
data = noise*(numpy.random.randn(*dshape) + 1j*numpy.random.randn(*dshape)) * scale_noise
if addToCol:
data+=colData[row0:(row0+nr)]
info(" $addToCol + noise --> CORRECTED_DATA (rows $row0 to %d)"%(row0+nr-1))
column = 'CORRECTED_DATA'
else : info("Adding noise to $column (rows $row0 to %d)"%(row0+nr-1))
tab.putcol(column,data,row0,nr);
tab.close()
def simcube (cube,nchan=None,npix=4096,cellsize=".5arcsec",niter=100000,padding=1.5,threshold=".2mJy",predict=True,dirty=False,restore=False,noise=0,resume=False,label=None,column='DATA',channelize=1,wprojplanes=0):
if label: v.LABEL = label
tab = ms.ms(subtable='SPECTRAL_WINDOW')
nchan = nchan or tab.getcol('NUM_CHAN')[0]
ms.CHANRANGE = 0,nchan-1;
imager.wprojplanes = wprojplanes;
imager.IMAGE_CHANNELIZE = 1;
if predict:
imager.predict_vis(image=cube,padding=padding,copy=False,column=column);
if noise > 0:
simnoise(addToCol=column,noise=noise)
if dirty:
info('Weights are%s'%WEIGHTS)
for weight in WEIGHTS.split(':'):
opts,weight_txt,quals = get_weight_opts(weight)
restore.update(opts)
dirty_image = II("${OUTFILE}-$weight.dirty.fits")
model_image = II("${OUTFILE}-$weight.model.fits")
residual_image = II("${OUTFILE}-$weight.residual.fits")
restored_image = II("${OUTFILE}-$weight.restored.fits")
imager.make_image(dirty=dirty,restore=restore,channelize=channelize,column=column if restore==False else "CORRECTED_DATA",dirty_image=dirty_image,model_image=model_image,residual_image=residual_image,restored_image=restored_image,**opts);
def azishe(config='$CFG'):
# Get parameters from json file
with open(II(config)) as jsn_std:
jparams = json.load(jsn_std)
# Remove empty strings and convert unicode characters to strings
params = {}
for key, val in jparams.iteritems():
# Make sure all keys are strings
_key = str(key)
# ignore empty strings and comments
if val == "" or _key == "#":
pass
# convert unicode values to strings
elif isinstance(val, unicode):
params[_key] = str(val).lower()
else:
params[_key] = val
get_opts = lambda prefix: filter(lambda a: a[0].startswith(prefix),
params.items())
# Retrieve MS and imager options
ms_dict = dict([(key.split('ms_')[-1], val)
for (key, val) in get_opts('ms_')])
im_dict = dict([(key.split('im_')[-1], val)
for (key, val) in get_opts('im_')])
# Seperate deconvolution settings
_deconv = {}
for dcv in 'lwimager wsclean moresane casa'.split():
if params[dcv]:
_deconv.update({
dcv:
dict([(key.split(dcv + '_')[-1], val)
for (key, val) in get_opts(dcv + '_')])
})
# Set imaging options
im.IMAGER = imager.IMAGER = params['imager'].lower()
for opt in 'npix weight robust mode stokes'.split():
if opt in im_dict.keys():
setattr(im, opt, im_dict.pop(opt))
im.cellsize = '%farcsec' % (im_dict.pop('cellsize'))
im.stokes = im.stokes.upper()
weight_fov = im_dict.pop('weight_fov')
if weight_fov:
im.weight_fov = '%farcmin' % weight_fov
# Create empty MS
synthesis = ms_dict.pop('synthesis')
scalenoise = 1
if synthesis > 12:
scalenoise = math.sqrt(12.0 / synthesis)
msname = II('rodrigues%f.MS' % (time.time()))
obs = params['observatory'].lower()
antennas = _OBS[obs]
freq0 = ms_dict.pop('freq0') * 1e6
dfreq = ms_dict.pop('dfreq') * 1e3
direction = "J2000,%s,%s" % (ms_dict.pop('ra'), ms_dict.pop('dec'))
ms.create_empty_ms(msname=msname,
synthesis=synthesis,
freq0=freq0,
dfreq=dfreq,
tel=obs,
pos='%s/%s' % (OBSDIR, antennas),
direction=direction,
**ms_dict)
if exists(msname):
v.MS = msname
else:
abort(
"Something went wrong while creating the empty MS. Please check logs for details"
)
makedir(DESTDIR)
ms.plot_uvcov(ms=.1,
width=10,
height=10,
dpi=150,
save="$OUTFILE-uvcov.png")
# Set noise for simulation
spwtab = ms.ms(subtable="SPECTRAL_WINDOW")
freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID, 0]
spwtab.close()
if params['add_noise']:
noise = compute_vis_noise(sefd=get_sefd(freq0)) * scalenoise
else:
noise = 0
# Simulate Visibilities
_katalog = params['katalog_id']
if _katalog:
katalog = '%s/%s' % (KATDIR, _KATALOG[params['katalog_id']])
lsmname = II('${OUTDIR>/}${MS:BASE}.lsm.html')
radius = float(params['radius'])
fluxrange = params['fluxrange'].split('-')
if len(fluxrange) > 1:
fluxrange = map(float, fluxrange)
elif len(fluxrange) == 1:
fluxrange = [0, float(fluxrange[0])]
select = ''
fits = verify_sky(katalog) == 'FITS'
if radius or fluxrange:
if radius: select += '--select="r<%fdeg" ' % radius
if fluxrange:
select += '--select="I<%f" ' % fluxrange[1]
select += '--select="I>%f" ' % fluxrange[0]
if not fits:
x.sh(
'tigger-convert $select --recenter=$direction $katalog $lsmname -f'
)
else:
from pyrap.measures import measures
dm = measures()
direction = dm.direction('J2000', ms_opts[ra], ms_opts[dec])
ra = np.rad2deg(direction['m0']['value'])
dec = np.rad2deg(direction['m1']['value'])
hdu = pyfits.open(temp_file)
hdu[0].hdr['CRVAL1'] = ra
hdu[0].hdr['CRVAL2'] = dec
hdu.writeto(tmp_file, clobber=True)
simsky(lsmname=lsmname,
tdlsec='turbo-sim:custom',
noise=noise,
column='CORRECTED_DATA')
skymodel = params['sky_model']
if skymodel:
simsky(lsmname=skymodel,
noise=0 if katalog else noise,
addToCol='CORRECTED_DATA')
## Finally Lets image
# make dirty map
im.make_image(psf=True, **im_dict)
# Deconvolve
for deconv in _deconv:
deconv = deconv.lower()
if deconv in STAND_ALONE_DECONV:
im.make_image(algorithm=deconv,
restore=_deconv[deconv],
restore_lsm=False,
**im_dict)
else:
im.IMAGER = deconv
im.make_image(dirty=False,
restore=_deconv[deconv],
restore_lsm=False,
**im_dict)
xo.sh('tar -czvf ${OUTDIR>/}${MS:BASE}.tar.gz $msname')
#!/usr/bin/env python
from rlist import randList
from ms import ms
from array_printer import print_array
with open('../size.txt','r') as f:
size = int(f.readline().strip())
pre_sort = randList(size)
post_sort = ms(pre_sort)
print_array(pre_sort)
print_array(post_sort)
def make_ifrgain_plots(filename="$STEFCAL_DIFFGAIN_SAVE",
prefix="IG",
feed="$IFRGAIN_PLOT_FEED",
msname="$MS"):
"""Makes a set of ifrgain plots from the specified saved file."""
import pylab
from Timba.Meq import meq
filename, prefix, msname = interpolate_locals("filename prefix msname")
global __IFRGAIN_PREFIX, _IFRGAIN_TYPE, _IFRGAIN_COMP
_IFRGAIN_PREFIX = prefix
dir = os.path.dirname(IFRGAIN_PLOT)
if dir:
makedir(dir)
# load baseline info, if MS is available
baseline = None
if msname:
try:
anttab = ms.ms(msname, "ANTENNA")
antpos = anttab.getcol("POSITION")
antname = anttab.getcol("NAME")
antpos = [(name, antpos[i]) for i, name in enumerate(antname)]
# make dict of p,q to baseline length
baseline = dict([("%s-%s" % (p, q),
math.sqrt(((ppos - qpos)**2).sum()))
for p, ppos in antpos for q, qpos in antpos])
except:
traceback.print_exc()
warn(
"can't access $msname antenna table, so IFR gain versus baseline length plots will not be made"
)
else:
warn(
"MS not specified, thus no antenna info. IFR gain versus baseline length plots will not be made"
)
# feed labels
feeds = ("RR", "LL", "RL",
"LR") if FEED_TYPE.upper() == "RL" else ("XX", "YY", "XY", "YX")
ig = pickle.load(open(filename, "rb"))
def plot_xy(content, title):
"""Plots x vs y"""
pylab.errorbar([x for l, (x, xe), (y, ye) in content],
[y for l, (x, xe), (y, ye) in content],
[ye for l, (x, xe), (y, ye) in content],
[xe for l, (x, xe), (y, ye) in content],
fmt=None,
ecolor="lightgrey")
# plot labels
for label, (x, xe), (y, ye) in content:
pylab.text(x,
y,
label,
horizontalalignment='center',
verticalalignment='center',
size=8)
pylab.title(title)
def plot_baseline(content, baseline, title, feeds):
"""Plots offset versus baseline"""
bl = []
xx = []
xxe = []
lab = []
col = []
for l, (x, xe), (y, ye) in content:
b = baseline.get(l, None)
if b is None:
warn("baseline $l not found in MS ANTENNA table")
else:
lab += ["%s:%s" % (l, feeds[0]),
"%s:%s" % (l, feeds[1])]
col += ["blue", "red"]
xx += [x, y]
xxe += [xe, ye]
bl += [b, b]
pylab.axhline(1, color='lightgrey')
pylab.errorbar(bl, xx, yerr=xxe, fmt=None, ecolor="lightgrey")
# plot labels
for l, x, y, c in zip(lab, bl, xx, col):
pylab.text(x,
y,
l,
horizontalalignment='center',
verticalalignment='center',
size=8,
color=c)
pylab.xlabel("Baseline, m.")
pylab.title(title)
def plot_hist(content, title):
"""Plots histogram"""
values = [x for l, (x, xe), (y, ye) in content
] + [y for l, (x, xe), (y, ye) in content]
hist = pylab.hist(values)
pylab.xlim(min(values), max(values))
pylab.title(title)
def plot_complex(content, title):
"""Plots x vs y"""
# plot errors bars, if available
pylab.axhline(0, color='lightgrey')
pylab.axvline(1, color='lightgrey')
pylab.errorbar([x.real for l1, l2, (x, xe), (y, ye) in content],
[x.imag for l1, l2, (x, xe), (y, ye) in content],
[xe for l1, l2, (x, xe), (y, ye) in content],
[xe for l1, l2, (x, xe), (y, ye) in content],
fmt=None,
ecolor="lightgrey")
pylab.errorbar([y.real for l1, l2, (x, xe), (y, ye) in content],
[y.imag for l1, l2, (x, xe), (y, ye) in content],
[ye for l1, l2, (x, xe), (y, ye) in content],
[ye for l1, l2, (x, xe), (y, ye) in content],
fmt=None,
ecolor="lightgrey")
# max plot amplitude -- max point plus 1/4th of the error bar
maxa = max(
[max(abs(x), abs(y)) for l1, l2, (x, xe), (y, ye) in content])
# plotlim = max([ abs(numpy.array([
# getattr(v,attr)+sign*e/4 for v,e in (x,xe),(y,ye) for attr in 'real','imag' for sign in 1,-1
# ])).max()
# for l1,l2,(x,xe),(y,ye) in content ]);
minre, maxre, minim, maxim = 2, -2, 2, -2
for l1, l2, (x, xe), (y, ye) in content:
offs = numpy.array([
getattr(v, attr) + sign * e / 4 for v, e in (x, xe), (y, ye)
for attr in 'real', 'imag' for sign in 1, -1
])
minre, maxre = min(x.real - xe / 4, y.real - ye / 4,
minre), max(x.real + xe / 4, y.real + ye / 4,
maxre)
minim, maxim = min(x.imag - xe / 4, y.imag - ye / 4,
minim), max(x.imag + xe / 4, y.imag + ye / 4,
maxim)
def get_data(msfile='$MS',src_name='$SRC',out_data_set_tag='$TAG'):
dataSetFolder = II('%s'%out_data_set_tag)
msname = II('%s'%msfile)
srcFolder = II('%s'%src_name)
x.sh("mkdir -p ./%s"%srcFolder)
x.sh("mkdir -p ./%s/%s"%(srcFolder,dataSetFolder))
tab = ms.ms(msname,write=False);
info("columns are",*(tab.colnames()));
uvw = tab.getcol("UVW")
spwtab = ms.ms(subtable="SPECTRAL_WINDOW")
freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID,0]#get the first freq
wavelength = LightSpeed/freq0
freqStepVect = spwtab.getcol("CHAN_WIDTH")
freqsVect=spwtab.getcol("CHAN_FREQ")
info(freqsVect)
#Specs
nchan = len(freqsVect[0,:]);
bandwidth = spwtab.getcol("CHAN_WIDTH")[ms.SPWID,0]
spwtab.close()
field = tab.getcol("FIELD_ID");
ant1=tab.getcol("ANTENNA1");
ant2=tab.getcol("ANTENNA2");
time = tab.getcol("TIME");
scantab = tab.getcol("SCAN_NUMBER")
integrationTimeVect = tab.getcol("EXPOSURE");
dt = tab.getcol("EXPOSURE",0,1)[0]
dtf = (tab.getcol("TIME",tab.nrows()-1,1)-tab.getcol("TIME",0,1))[0]
info(">>>> Freq %.2f MHz (lambda=%.2fm), bandwidth %.2g kHz, %.2fs integrations, %.2fh synthesis"%(freq0*1e-6,wavelength,bandwidth*1e-3,dt,dtf/3600))
sio.savemat("./%s/%s/msSpecs.mat"%(srcFolder,dataSetFolder),{'field':field,'ant1':ant1.astype(float),'ant2':ant2.astype(float),'freqsVect':freqsVect,'time':time,'uvw':uvw,'scans':scantab})
#Data
weightfull = tab.getcol("WEIGHT_SPECTRUM");
flag_row = tab.getcol("FLAG_ROW")
flagAll= tab.getcol("FLAG")
data_full = tab.getcol("DATA");
for ifreq in range(0, nchan):
data_selected = data_full[:,ifreq,:];
weight_selected = weightfull[:,ifreq,:];
weightI= (weight_selected[:,0]+weight_selected[:,3])
dataI_w = (weight_selected[:,0]*data_selected[:,0]+weight_selected[:,3]*data_selected[:,3])/weightI ;
flag_bis = numpy.logical_or(flagAll[:,ifreq,0],flagAll[:,ifreq,3]);
flag = numpy.logical_or(flag_bis,flag_row)
flag = flag.astype(float)
sio.savemat("./%s/%s/data.mat"%(srcFolder,dataSetFolder),{'y_I':dataI_w, 'weights':weightI.astype(float), 'flag':flag});
info("Read data .. writing file..Freq %s"%(ifreq))
tab.close()
def get_field_center():
t = ms.ms(subtable="FIELD")
phase_centre = (t.getcol("PHASE_DIR"))[0,0,:]
t.close()
return phase_centre[0], phase_centre[1] #ra0,dec0 in radians
def azishe():
jdict = readJson(CONFIG)
add = jdict.get("add_to_column", False)
addnoise = jdict.get("addnoise", False)
sefd = jdict.get("sefd", 500)
v.MS = "{:s}/{:s}".format(MSDIR, jdict["msname"])
v.LOG = II("${OUTDIR>/}log-${MS:BASE}-simulation.txt")
column = jdict.get("column", "DATA")
copy = jdict.get("copy_to_CORRECTED_DATA", False)
newcol = jdict.get("custom_data_column", None)
if newcol:
im.argo.addcol(newcol)
if jdict["skymodel"] in [None, False] and addnoise:
noise = compute_vis_noise(sefd)
simnoise(noise, column=column)
if copy and column!="CORRECTED_DATA":
ms.copycol(fromcol=column, tocol="CORRECTED_DATA")
return
for item in ["/input/", "/data/skymodels/"]:
lsmname = "{:s}/{:s}".format(item, jdict["skymodel"])
if os.path.exists(lsmname):
break
v.LSM = II("${lsmname:FILE}")
std.copy(lsmname, LSM)
if jdict.get("recenter", False):
direction = jdict.get("direction", False)
if direction:
ftab = ms.ms(subtable="FIELD")
ra,dec = ftab.getcol("PHASE_DIR")[jdict.get("field_id",0)][0]
direction = "J2000,%frad,%frad"%(ra,dec)
x.sh("tigger-convert --recenter=$direction $LSM -f")
options = {}
if addnoise:
noise = compute_vis_noise(sefd)
options["noise_stddev"] = noise
beam = jdict.get("Ejones", False)
if beam:
options["me.e_enable"] = 1
options["me.p_enable"] = 1
options["me.e_module"] = "Siamese_OMS_pybeams_fits"
options["me.e_advanced"] = 1
options["me.e_all_stations"] = 1
options["pybeams_fits.l_axis"] = jdict.get("beam_l_axis", "L")
options["pybeams_fits.m_axis"] = jdict.get("beam_m_axis", "M")
options["pybeams_fits.filename_pattern"] = "%s/%s"%(INDIR, jdict["beam_files_pattern"])
rms_perr = jdict.get("pointing_accuracy", 0)
# Include pointing errors if needed
if rms_perr:
anttab = ms.ms(subtable="ANTENNA")
NANT = anttab.nrows()
options["me.epe_enable"] = 1
perr = numpy.random.randn(NANT)*rms_perr, numpy.random.randn(NANT)*rms_perr
ll, mm = " ".join( map(str, perr[0]) ), " ".join( map(str, perr[-1]) )
options['oms_pointing_errors.pe_l.values_str'] = "'%s'"%ll
options['oms_pointing_errors.pe_m.values_str'] = "'%s'"%mm
_section = dict(sim = "sim",
add_G = "sim:G")
if jdict.get("gjones", False):
section = "add_G"
else:
section = "sim"
mode = '"add to MS"' if add else '"sim only"'
options["sim_mode"] = mode
options["ms_sel.input_column"] = add
options["ms_sel.output_column"] = column
mqt.msrun(II("${mqt.CATTERY}/Siamese/turbo-sim.py"),
job = '_tdl_job_1_simulate_MS',
section = _section[section],
options = options,
args = ["${lsm.LSM_TDL}"])
if copy and column!="CORRECTED_DATA":
ms.copycol(fromcol=column, tocol="CORRECTED_DATA")
def azishe(config='$CFG'):
# Get parameters from json file
with open(II(config)) as jsn_std:
jparams = json.load(jsn_std)
# Remove empty strings and convert unicode characters to strings
params = {}
for key, val in jparams.iteritems():
# Make sure all keys are strings
_key = str(key)
# ignore empty strings and comments
if val=="" or _key=="#":
pass
# convert unicode values to strings
elif isinstance(val,unicode):
params[_key] = str(val).lower()
else:
params[_key] = val
get_opts = lambda prefix: filter(lambda a: a[0].startswith(prefix), params.items())
# Retrieve MS and imager options
ms_dict = dict([(key.split('ms_')[-1],val) for (key,val) in get_opts('ms_') ] )
im_dict = dict([(key.split('im_')[-1],val) for (key,val) in get_opts('im_') ] )
# Seperate deconvolution settings
_deconv = {}
for dcv in 'lwimager wsclean moresane casa'.split():
if params[dcv]:
_deconv.update( {dcv:dict([(key.split(dcv+'_')[-1],val) for (key,val) in get_opts(dcv+'_') ] )} )
# Set imaging options
im.IMAGER = imager.IMAGER = params['imager'].lower()
for opt in 'npix weight robust mode stokes'.split():
if opt in im_dict.keys():
setattr(im,opt,im_dict.pop(opt))
im.cellsize = '%farcsec'%(im_dict.pop('cellsize'))
im.stokes = im.stokes.upper()
weight_fov = im_dict.pop('weight_fov')
if weight_fov:
im.weight_fov = '%farcmin'%weight_fov
# Create empty MS
synthesis = ms_dict.pop('synthesis')
scalenoise = 1
if synthesis > 12:
scalenoise = math.sqrt(12.0/synthesis)
msname = II('rodrigues%f.MS'%(time.time()))
obs = params['observatory'].lower()
antennas = _OBS[obs]
freq0 = ms_dict.pop('freq0')*1e6
dfreq = ms_dict.pop('dfreq')*1e3
direction = "J2000,%s,%s"%(ms_dict.pop('ra'),ms_dict.pop('dec'))
ms.create_empty_ms(msname=msname,synthesis=synthesis,freq0=freq0,
dfreq=dfreq,tel=obs,pos='%s/%s'%(OBSDIR,antennas),
direction=direction,**ms_dict)
if exists(msname):
v.MS = msname
else:
abort("Something went wrong while creating the empty MS. Please check logs for details")
makedir(DESTDIR)
ms.plot_uvcov(ms=.1,width=10,height=10,dpi=150,save="$OUTFILE-uvcov.png")
# Set noise for simulation
spwtab = ms.ms(subtable="SPECTRAL_WINDOW")
freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID,0]
spwtab.close()
if params['add_noise']:
noise = compute_vis_noise(sefd=get_sefd(freq0)) * scalenoise
else:
noise = 0
# Simulate Visibilities
_katalog = params['katalog_id']
if _katalog:
katalog = '%s/%s'%(KATDIR,_KATALOG[params['katalog_id']])
lsmname = II('${OUTDIR>/}${MS:BASE}.lsm.html')
radius = float(params['radius'])
fluxrange = params['fluxrange'].split('-')
if len(fluxrange)>1:
fluxrange = map(float,fluxrange)
elif len(fluxrange)==1:
fluxrange = [0,float(fluxrange[0])]
select = ''
fits = verify_sky(katalog) == 'FITS'
if radius or fluxrange:
if radius: select += '--select="r<%fdeg" '%radius
if fluxrange:
select += '--select="I<%f" '%fluxrange[1]
select += '--select="I>%f" '%fluxrange[0]
if not fits:
x.sh('tigger-convert $select --recenter=$direction $katalog $lsmname -f')
else:
from pyrap.measures import measures
dm = measures()
direction = dm.direction('J2000',ms_opts[ra],ms_opts[dec])
ra = np.rad2deg(direction['m0']['value'])
dec = np.rad2deg(direction['m1']['value'])
hdu = pyfits.open(temp_file)
hdu[0].hdr['CRVAL1'] = ra
hdu[0].hdr['CRVAL2'] = dec
hdu.writeto(tmp_file,clobber=True)
simsky(lsmname=lsmname,tdlsec='turbo-sim:custom',noise=noise,column='CORRECTED_DATA')
skymodel = params['sky_model']
if skymodel:
simsky(lsmname=skymodel,noise=0 if katalog else noise,addToCol='CORRECTED_DATA')
## Finally Lets image
# make dirty map
im.make_image(psf=True,**im_dict)
# Deconvolve
for deconv in _deconv:
deconv = deconv.lower()
if deconv in STAND_ALONE_DECONV:
im.make_image(algorithm=deconv,restore=_deconv[deconv],restore_lsm=False,**im_dict)
else:
im.IMAGER = deconv
im.make_image(dirty=False,restore=_deconv[deconv],restore_lsm=False,**im_dict)
xo.sh('tar -czvf ${OUTDIR>/}${MS:BASE}.tar.gz $msname')
def make_ifrgain_plots (filename="$STEFCAL_DIFFGAIN_SAVE",prefix="IG",feed="$IFRGAIN_PLOT_FEED",msname="$MS"):
"""Makes a set of ifrgain plots from the specified saved file."""
import pylab
from Timba.Meq import meq
filename,prefix,msname = interpolate_locals("filename prefix msname");
global __IFRGAIN_PREFIX,_IFRGAIN_TYPE,_IFRGAIN_COMP;
_IFRGAIN_PREFIX = prefix;
dir = os.path.dirname(IFRGAIN_PLOT);
if dir:
makedir(dir)
# load baseline info, if MS is available
baseline = None;
if msname:
try:
anttab = ms.ms(msname,"ANTENNA");
antpos = anttab.getcol("POSITION");
antname = anttab.getcol("NAME");
antpos = [ (name,antpos[i]) for i,name in enumerate(antname) ];
# make dict of p,q to baseline length
baseline = dict([ ("%s-%s"%(p,q),math.sqrt(((ppos-qpos)**2).sum())) for p,ppos in antpos for q,qpos in antpos ])
except:
traceback.print_exc();
warn("can't access $msname antenna table, so IFR gain versus baseline length plots will not be made");
else:
warn("MS not specified, thus no antenna info. IFR gain versus baseline length plots will not be made");
# feed labels
feeds = ("RR","LL","RL","LR") if FEED_TYPE.upper() == "RL" else ("XX","YY","XY","YX");
ig = cPickle.load(file(filename))
def plot_xy (content,title):
"""Plots x vs y""";
pylab.errorbar(
[x for l,(x,xe),(y,ye) in content],[y for l,(x,xe),(y,ye) in content],
[ye for l,(x,xe),(y,ye) in content],[xe for l,(x,xe),(y,ye) in content],
fmt=None,ecolor="lightgrey"
);
# plot labels
for label,(x,xe),(y,ye) in content:
pylab.text(x,y,label,horizontalalignment='center',verticalalignment='center',size=8)
pylab.title(title)
def plot_baseline (content,baseline,title,feeds):
"""Plots offset versus baseline""";
bl = [];
xx = [];
xxe = [];
lab = [];
col = [];
for l,(x,xe),(y,ye) in content:
b = baseline.get(l,None);
if b is None:
warn("baseline $l not found in MS ANTENNA table")
else:
lab += [ "%s:%s"%(l,feeds[0]),"%s:%s"%(l,feeds[1]) ];
col += [ "blue","red" ]
xx += [x,y];
xxe += [xe,ye];
bl += [b,b];
pylab.axhline(1,color='lightgrey')
pylab.errorbar(bl,xx,yerr=xxe,fmt=None,ecolor="lightgrey");
# plot labels
for l,x,y,c in zip(lab,bl,xx,col):
pylab.text(x,y,l,horizontalalignment='center',verticalalignment='center',size=8,color=c)
pylab.xlabel("Baseline, m.");
pylab.title(title)
def plot_hist (content,title):
"""Plots histogram""";
values = [x for l,(x,xe),(y,ye) in content] + [y for l,(x,xe),(y,ye) in content];
hist = pylab.hist(values);
pylab.xlim(min(values),max(values));
pylab.title(title)
def plot_complex (content,title):
"""Plots x vs y""";
# plot errors bars, if available
pylab.axhline(0,color='lightgrey')
pylab.axvline(1,color='lightgrey')
pylab.errorbar(
[x.real for l1,l2,(x,xe),(y,ye) in content],[x.imag for l1,l2,(x,xe),(y,ye) in content],
[xe for l1,l2,(x,xe),(y,ye) in content],[xe for l1,l2,(x,xe),(y,ye) in content],
fmt=None,ecolor="lightgrey"
);
pylab.errorbar(
[y.real for l1,l2,(x,xe),(y,ye) in content],[y.imag for l1,l2,(x,xe),(y,ye) in content],
[ye for l1,l2,(x,xe),(y,ye) in content],[ye for l1,l2,(x,xe),(y,ye) in content],
fmt=None,ecolor="lightgrey"
);
# max plot amplitude -- max point plus 1/4th of the error bar
maxa = max([ max(abs(x),abs(y)) for l1,l2,(x,xe),(y,ye) in content ]);
# plotlim = max([ abs(numpy.array([
# getattr(v,attr)+sign*e/4 for v,e in (x,xe),(y,ye) for attr in 'real','imag' for sign in 1,-1
# ])).max()
# for l1,l2,(x,xe),(y,ye) in content ]);
minre, maxre, minim, maxim = 2, -2, 2, -2
for l1,l2,(x,xe),(y,ye) in content:
offs = numpy.array([ getattr(v,attr)+sign*e/4 for v,e in (x,xe),(y,ye)
for attr in 'real','imag' for sign in 1,-1 ])
minre, maxre = min(x.real-xe/4, y.real-ye/4, minre), max(x.real+xe/4, y.real+ye/4, maxre)
minim, maxim = min(x.imag-xe/4, y.imag-ye/4, minim), max(x.imag+xe/4, y.imag+ye/4, maxim)
def residual():
t = ms.ms(write=True)
corr_data = t.getcol("CORRECTED_DATA")
data = t.getcol("DATA")
t.putcol("CORRECTED_DATA",corr_data-data)
t.close()
