def noneg_model(modelname, ms, imagename, **kwargs):
"""
Given a model image, set all model components positive, then ft them into
the ms's model column
"""
if os.path.exists(modelname + ".positive"):
shutil.rmtree(modelname + ".positive")
immath(
imagename=modelname,
expr='iif(IM0<0, 0.0, IM0)',
outfile=modelname + ".positive",
)
if os.path.exists(modelname):
if os.path.exists(modelname + ".old"):
shutil.rmtree(modelname + ".old")
os.rename(modelname, modelname + ".old")
tclean(vis=ms,
imagename=imagename,
startmodel=modelname + ".positive",
niter=0,
deconvolver='mtmfs',
specmode='mfs',
nterms=1,
calcpsf=False,
calcres=False,
interactive=False,
savemodel='modelcolumn',
**kwargs)
def fd_images(vis, cleanup=False, niter=None, spws=['0~1', '2~5', '6~10', '11~20', '21~30', '31~43'], imgoutdir='./',
bright=None, stokes="XX"):
''' Create standard full-disk images in "images" subdirectory of the current directory.
If cleanup is True, delete those images after completion, leaving only the fits images.
'''
# Check if "images" directory exists (if not, create it and mark it for later deletion)
try:
if os.stat('images'):
rm_images = False # Mark as not removeable
except:
os.mkdir('images')
if cleanup:
rm_images = True # Mark as removeable
else:
rm_images = False # Mark as not removeable
trange = ant_trange(vis)
tdate = trange.replace('/', '')[:8]
if niter is None:
niter = 5000
if bright is None:
bright = [True] * len(spws)
imagefile = []
fitsfile = []
for s, sp in enumerate(spws):
if bright[s]:
spwstr = '-'.join(['{:02d}'.format(int(sp_)) for sp_ in sp.split('~')])
imname = "images/briggs" + spwstr
# tclean(vis=vis, selectdata=True, spw=sp, timerange=trange,
# antenna="0~12", datacolumn="corrected", imagename=imname, imsize=[1024], cell=['2.5arcsec'],
# stokes="XX", projection="SIN", specmode="mfs", interpolation="linear", deconvolver="multiscale",
# scales=[0, 5, 15, 30], nterms=2, smallscalebias=0.6, restoration=True, weighting="briggs", robust=0,
# niter=niter, gain=0.05, savemodel="none")
os.system('rm -rf {}.*'.format(imname))
tclean(vis=vis, selectdata=True, spw=sp, timerange=trange,
antenna="0~12", datacolumn="data", imagename=imname, imsize=[1024], cell=['2.5arcsec'],
stokes=stokes, projection="SIN", specmode="mfs", interpolation="linear", deconvolver="multiscale",
scales=[0, 5, 15, 30], nterms=2, smallscalebias=0.6, restoration=True, weighting="briggs", robust=0,
niter=niter, gain=0.05, savemodel="none", usemask='auto-multithresh', pbmask=0.0,
sidelobethreshold=1.0, noisethreshold=2.5, lownoisethreshold=1.5, negativethreshold=5.0,
smoothfactor=1.0, minbeamfrac=0.3, cutthreshold=0.01, growiterations=75, dogrowprune=True,
minpercentchange=-1.0)
outfits = os.path.join(imgoutdir, 'eovsa_' + tdate + '.spw' + spwstr + '.tb.fits')
if os.path.exists(outfits):
os.system('rm -rf {}'.format(outfits))
imagefile.append(imname + '.image')
fitsfile.append(outfits)
hf.imreg(vis=vis, imagefile=imagefile, fitsfile=fitsfile, timerange=[trange] * len(fitsfile), toTb=True,
usephacenter=False, overwrite=True)
if rm_images:
shutil.rmtree('images') # Remove all images and the folder named images
# To add disk model image to the images, I can try scipy.ndimage routines gaussian_filter() and zoom()
return fitsfile
imagename = '18A-229_Q_mosaic_selfcal_iter0_dirty'
if not os.path.exists(imagename+".image.tt0.pbcor"):
# do a full-mosaic clean to enable mask creation
tclean(
vis=cont_vis,
spw='',
field="Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q,Sgr B2 S Q",
phasecenter='J2000 17h47m19.693 -28d23m11.527',
imsize=[9000,9000],
cell='0.02arcsec',
imagename=imagename,
niter=0,
threshold='1000mJy',
robust=0.5,
gridder='mosaic',
scales=[0,3,9,27],
deconvolver='mtmfs',
specmode='mfs',
nterms=2,
weighting='briggs',
pblimit=0.2,
interactive=False,
outframe='LSRK',
savemodel='none',
)
makefits(imagename)
# create a mask based on region selection (no thresholding here)
dirtyimagename = imagename+".image.tt0.pbcor"
def clean_iter(
tim, vis, imageprefix, imagesuffix, twidth, doreg, usephacenter,
reftime, ephem, msinfo, toTb, overwrite, selectdata, field, spw,
uvrange, antenna, scan, observation, intent, datacolumn, imsize, cell,
phasecenter, stokes, projection, startmodel, specmode, reffreq, nchan,
start, width, outframe, veltype, restfreq, interpolation, gridder,
facets, chanchunks, wprojplanes, vptable, usepointing, mosweight,
aterm, psterm, wbawp, conjbeams, cfcache, computepastep, rotatepastep,
pblimit, normtype, deconvolver, scales, nterms, smallscalebias,
restoration, restoringbeam, pbcor, outlierfile, weighting, robust,
npixels, uvtaper, niter, gain, threshold, nsigma, cycleniter,
cyclefactor, minpsffraction, maxpsffraction, interactive, usemask,
mask, pbmask, sidelobethreshold, noisethreshold, lownoisethreshold,
negativethreshold, smoothfactor, minbeamfrac, cutthreshold,
growiterations, dogrowprune, minpercentchange, verbose, restart,
savemodel, calcres, calcpsf, parallel, subregion, tmpdir, btidx):
from tclean_cli import tclean_cli as tclean
from split_cli import split_cli as split
bt = btidx # 0
if bt + twidth < len(tim) - 1:
et = btidx + twidth - 1
else:
et = len(tim) - 1
if bt == 0:
bt_d = tim[bt] - ((tim[bt + 1] - tim[bt]) / 2)
else:
bt_d = tim[bt] - ((tim[bt] - tim[bt - 1]) / 2)
if et == (len(tim) - 1) or et == -1:
et_d = tim[et] + ((tim[et] - tim[et - 1]) / 2)
else:
et_d = tim[et] + ((tim[et + 1] - tim[et]) / 2)
timerange = qa.time(qa.quantity(bt_d, 's'), prec=9, form='ymd')[0] + '~' + \
qa.time(qa.quantity(et_d, 's'), prec=9, form='ymd')[0]
btstr = qa.time(qa.quantity(bt_d, 's'), prec=9, form='fits')[0]
etstr = qa.time(qa.quantity(et_d, 's'), prec=9, form='fits')[0]
print('cleaning timerange: ' + timerange)
image0 = btstr.replace(':', '').replace('-', '')
imname = imageprefix + image0 + imagesuffix
# ms_tmp = tmpdir + image0 + '.ms'
# print('checkpoint 1')
# # split(vis=vis, outputvis=ms_tmp, field=field, scan=scan, antenna=antenna, timerange=timerange,
# # datacolumn=datacolumn)
# ms.open(vis)
# print('checkpoint 1-1')
# ms.split(ms_tmp,field=field, scan=scan, baseline=antenna, time=timerange,whichcol=datacolumn)
# print('checkpoint 1-2')
# ms.close()
# print('checkpoint 2')
if overwrite or (len(glob.glob(imname + '*')) == 0):
os.system('rm -rf {}*'.format(imname))
try:
tclean(vis=vis,
selectdata=selectdata,
field=field,
spw=spw,
timerange=timerange,
uvrange=uvrange,
antenna=antenna,
scan=scan,
observation=observation,
intent=intent,
datacolumn=datacolumn,
imagename=imname,
imsize=imsize,
cell=cell,
phasecenter=phasecenter,
stokes=stokes,
projection=projection,
startmodel=startmodel,
specmode=specmode,
reffreq=reffreq,
nchan=nchan,
start=start,
width=width,
outframe=outframe,
veltype=veltype,
restfreq=restfreq,
interpolation=interpolation,
gridder=gridder,
facets=facets,
chanchunks=chanchunks,
wprojplanes=wprojplanes,
vptable=vptable,
usepointing=usepointing,
mosweight=mosweight,
aterm=aterm,
psterm=psterm,
wbawp=wbawp,
conjbeams=conjbeams,
cfcache=cfcache,
computepastep=computepastep,
rotatepastep=rotatepastep,
pblimit=pblimit,
normtype=normtype,
deconvolver=deconvolver,
scales=scales,
nterms=nterms,
smallscalebias=smallscalebias,
restoration=restoration,
restoringbeam=restoringbeam,
pbcor=pbcor,
outlierfile=outlierfile,
weighting=weighting,
robust=robust,
npixels=npixels,
uvtaper=uvtaper,
niter=niter,
gain=gain,
threshold=threshold,
nsigma=nsigma,
cycleniter=cycleniter,
cyclefactor=cyclefactor,
minpsffraction=minpsffraction,
maxpsffraction=maxpsffraction,
interactive=interactive,
usemask=usemask,
mask=mask,
pbmask=pbmask,
sidelobethreshold=sidelobethreshold,
noisethreshold=noisethreshold,
lownoisethreshold=lownoisethreshold,
negativethreshold=negativethreshold,
smoothfactor=smoothfactor,
minbeamfrac=minbeamfrac,
cutthreshold=cutthreshold,
growiterations=growiterations,
dogrowprune=dogrowprune,
minpercentchange=minpercentchange,
verbose=verbose,
restart=restart,
savemodel=savemodel,
calcres=calcres,
calcpsf=calcpsf,
parallel=parallel)
# print('checkpoint 3')
if pbcor:
clnjunks = [
'.flux', '.mask', '.model', '.psf', '.residual', '.pb',
'.sumwt', '.image'
]
else:
clnjunks = [
'.flux', '.mask', '.model', '.psf', '.residual', '.pb',
'.sumwt', '.image.pbcor'
]
for clnjunk in clnjunks:
if os.path.exists(imname + clnjunk):
shutil.rmtree(imname + clnjunk)
if pbcor:
os.system('mv {} {}'.format(imname + '.image.pbcor',
imname + '.image'))
except:
print('error in cleaning image: ' + btstr)
return [False, btstr, etstr, '']
else:
print(imname + ' exists. Clean task aborted.')
if doreg and not os.path.isfile(imname + '.fits'):
# ephem.keys()
# msinfo.keys()
try:
# check if ephemfile and msinfofile exist
if not ephem:
print(
"ephemeris info does not exist, querying from JPL Horizons on the fly"
)
ephem = hf.read_horizons(vis=vis)
if not msinfo:
print("ms info not provided, generating one on the fly")
msinfo = hf.read_msinfo(vis)
hf.imreg(vis=vis,
ephem=ephem,
msinfo=msinfo,
timerange=timerange,
reftime=reftime,
imagefile=imname + '.image',
fitsfile=imname + '.fits',
toTb=toTb,
scl100=False,
usephacenter=usephacenter,
subregion=subregion)
if os.path.exists(imname + '.fits'):
shutil.rmtree(imname + '.image')
return [True, btstr, etstr, imname + '.fits']
else:
return [False, btstr, etstr, '']
except:
print('error in registering image: ' + btstr)
return [False, btstr, etstr, imname + '.image']
else:
if os.path.exists(imname + '.image'):
return [True, btstr, etstr, imname + '.image']
else:
return [False, btstr, etstr, '']
def checksource(overwrite=True, verbose=False, subdir='', splitcal_vis=''):
"""
Images the phasecal and check source in a manually-calibrated dataset and
reports statistics. Expects to find the *.split.cal measurement set and
the corresponding .fluxscale file for it.
Inputs:
overwrite: if True, overwrite any existing image from a previous execution
splitcal_vis: defaults to *.cal, but can be specified as list of strings,
or a comma-delimited string
Outputs:
png image plots of each calibrator, and an ASCII file for each dataset
The name of the ASCII file, and a list of pngs are returned.
"""
# Read the dataset(s) and get properties
if (splitcal_vis == ''):
vislist = glob.glob('*.cal')
else:
if (type(splitcal_vis) == str):
vislist = splitcal_vis.split(',')
else:
vislist = splitcal_vis
print("Checking datasets: ", vislist)
mymsmd = au.createCasaTool(msmdtool)
if (len(subdir) > 0):
if (os.path.exists(subdir)):
if (subdir[-1] != '/'):
subdir += '/'
else:
os.mkdir(subdir)
if (subdir[-1] != '/'):
subdir += '/'
pnglist = []
textfiles = []
for vis in vislist:
mymsmd.open(vis)
freq = mymsmd.meanfreq(0, unit='GHz')
# Check Source
check = mymsmd.fieldsforintent('OBSERVE_CHECK_SOURCE*', True)[0]
checkid = mymsmd.fieldsforintent('OBSERVE_CHECK_SOURCE*', False)[0]
checkpos = mymsmd.phasecenter(checkid)
# Phase calibrator
phase = mymsmd.fieldsforintent('CALIBRATE_PHASE*', True)[0]
phaseid = mymsmd.fieldsforintent('CALIBRATE_PHASE*', False)[0]
phasepos = mymsmd.phasecenter(phaseid)
if ('OBSERVE_TARGET#ON_SOURCE' in mymsmd.intents()):
nScienceFields = len(
mymsmd.fieldsforintent('OBSERVE_TARGET*', False))
science = mymsmd.fieldsforintent('OBSERVE_TARGET*', True)[0]
scienceid = mymsmd.fieldsforintent('OBSERVE_TARGET*', False)[0]
else:
nScienceFields = 0
mymsmd.done()
floatcell = au.pickCellSize(vis,
maxBaselinePercentile=99,
verbose=verbose)
cell = au.pickCellSize(vis,
maxBaselinePercentile=99,
cellstring=True,
verbose=verbose)
# imsize = int(au.nextValidImsize(int(5.0/floatcell))) # valid when we only had checksources for synthBeam < 0.25
imsize = int(
au.nextValidImsize(
int(
np.max([5.0, 5.0 * au.estimateSynthesizedBeam(vis)]) /
floatcell)))
print("imsize = ", imsize)
region = 'circle[[%dpix , %dpix], 15pix ]' % (int(
imsize / 2), int(imsize / 2))
if False:
# original method (for bands 3-6 only)
cell = str(np.round(0.015 * (100 / freq), 3)) + 'arcsec'
if freq < 116.0:
imsize = [320, 320]
region = 'circle[[160pix , 160pix] ,15pix ]'
else:
imsize = [680, 680]
region = 'circle[[340pix , 340pix] ,15pix ]'
###################################
# IMAGE
###################################
weighting = 'briggs'
robust = 0.5
niter = 50
threshold = '0.0mJy'
spw = ''
separation = au.angularSeparationOfTwoFields(vis, checkid, phaseid)
if (nScienceFields > 0):
separation_pcal_science = au.angularSeparationOfTwoFields(
vis, scienceid, phaseid)
separation_check_science = au.angularSeparationOfTwoFields(
vis, scienceid, checkid)
fieldtype = ['checksource', 'phasecal']
field = [check, phase]
for i, cal in enumerate(field):
if (not os.path.exists(cal + '_' + vis + '.image') or overwrite):
os.system('rm -rf ' + cal + '_' + vis + '.*')
if verbose:
print(
"Running tclean('%s', field='%s', cell=%s, imsize=%s, ...)"
% (vis, cal, str(cell), str(imsize)))
tclean(vis=vis,
imagename=cal + '_' + vis,
field=cal,
spw=spw,
specmode='mfs',
deconvolver='hogbom',
imsize=imsize,
cell=cell,
weighting=weighting,
robust=robust,
niter=niter,
threshold=threshold,
interactive=False,
mask=region,
gridder='standard')
png = subdir + fieldtype[i] + '_' + cal + '_' + vis + '.image.png'
pnglist.append(png)
au.imviewField(cal + '_' + vis + '.image',
radius=30 * floatcell,
contourImage=cal + '_' + vis + '.mask',
levels=[1],
plotfile=png)
###################################
# ANALYZE
###################################
###########
# PHASE
###########
imagename = phase + '_' + vis
if verbose:
print("Running imfit('%s', region='%s')" %
(imagename + '.image', region))
# Fit the phase source to get position and flux
imagefit = imfit(imagename=imagename + '.image', region=region)
fitresults = au.imfitparse(imagefit)
# Compare the Positions
phasepos_obs = au.direction2radec(phasepos)
if fitresults is not None:
phasepos_fit = ','.join(fitresults.split()[:2])
phasepos_diff = au.angularSeparationOfStrings(
phasepos_obs, phasepos_fit, verbose=False) * 3600.
# Compare the Flux densities
peakIntensity = au.imagePeak(imagename + '.image')
selffluxfile = glob.glob('*.fluxscale')[0]
fluxscaleResult = au.fluxscaleParseLog(selffluxfile, field=phase)
if fluxscaleResult is not None:
selfflux = fluxscaleResult[0][0]
phaseflux_fit = float(fitresults.split()[2])
phaseCoherence = 100 * peakIntensity / phaseflux_fit
phaseflux_diff = 100 * (selfflux - phaseflux_fit) / selfflux
# Print the final results and save to file
textfile = subdir + 'calimage_results_' + vis + '.txt'
textfiles.append(textfile)
f = open(textfile, 'w')
f.write(
'\n*************************************************************\n\n'
)
line = 'CHECK_SOURCE IMAGE ANALYSIS REPORT (version %s)\n' % version(
short=True)
writeOut(f, line)
info = au.getFitsBeam(imagename + '.image')
synthBeam = (info[0] * info[1])**0.5
if fitresults is None:
line = "Phasecal %s: imfit failed" % (phase)
elif fluxscaleResult is not None:
line = "Phasecal %s: Position difference = %s arcsec = %s synth.beam, Flux %% difference = %s" % (
phase, au.roundFiguresToString(phasepos_diff, 3),
au.roundFiguresToString(phasepos_diff / synthBeam, 3),
au.roundFiguresToString(phaseflux_diff, 3))
writeOut(f, line)
line = " coherence = peakIntensity/fittedFluxDensity = %s%%" % (
au.roundFiguresToString(phaseCoherence, 3))
else:
line = "Phasecal %s: Position difference = %s arcsec = %s synth.beam" % (
phase, au.roundFiguresToString(phasepos_diff, 3),
au.roundFiguresToString(phasepos_diff / synthBeam, 3))
writeOut(f, line)
f.close()
if fluxscaleResult is None:
print(
"Full checksource analysis is not supported if there is no flux calibrator"
)
return textfiles, pnglist
###########
# CHECK
###########
imagename = check + '_' + vis
# Fit the check source to get position and flux
if verbose:
print("Running imfit('%s', region='%s')" %
(imagename + '.image', region))
imagefit = imfit(imagename=imagename + '.image', region=region)
fitresults = au.imfitparse(imagefit, deconvolved=True)
info = au.getFitsBeam(imagename + '.image')
synthMajor, synthMinor = info[0:2]
synthBeam = (info[0] * info[1])**0.5
# Compare the Positions
checkpos_obs = au.direction2radec(checkpos)
if fitresults is not None:
checkpos_fit = ','.join(fitresults.split()[:2])
checkpos_diff = au.angularSeparationOfStrings(
checkpos_obs, checkpos_fit, verbose=False) * 3600.
# Compare the Flux densities
selffluxfile = glob.glob('*.fluxscale')[0]
results = au.fluxscaleParseLog(selffluxfile, field=check)
peakIntensity = au.imagePeak(imagename + '.image')
if (results is not None and fitresults is not None):
selfflux = results[0][0]
checkflux_fit = float(fitresults.split()[2])
checkflux_diff = 100 * (selfflux - checkflux_fit) / selfflux
checkCoherence = 100 * peakIntensity / checkflux_fit
if fitresults is not None:
if verbose:
print("Checksource fitresults: ", fitresults)
deconvolvedMajor = float(fitresults.split()[5])
deconvolvedMinor = float(fitresults.split()[7])
# Print the final results and save to file
f = open(textfile, 'a')
if fitresults is None:
line = "Checksource %s: imfit failed" % (phase)
else:
if (results is not None):
line = "\nChecksource %s: Position difference = %s arcsec = %s synth.beam, Flux %% difference = %s" % (
check, au.roundFiguresToString(checkpos_diff, 3),
au.roundFiguresToString(checkpos_diff / synthBeam, 3),
au.roundFiguresToString(checkflux_diff, 3))
writeOut(f, line)
line = " coherence = peakIntensity/fittedFluxDensity = %s%%" % (
au.roundFiguresToString(checkCoherence, 3))
else:
line = "\nChecksource %s: Position difference = %s arcsec = %s synth.beam" % (
check, au.roundFiguresToString(checkpos_diff, 3),
au.roundFiguresToString(checkpos_diff / synthBeam, 3))
writeOut(f, line)
line = " beam size = %s x %s arcsec" % (au.roundFiguresToString(
synthMajor, 3), au.roundFiguresToString(synthMinor, 3))
writeOut(f, line)
line = " apparent deconvolved size = %s x %s arcsec = %s synth.beam area" % (
au.roundFiguresToString(deconvolvedMajor, 2),
au.roundFiguresToString(deconvolvedMinor, 2),
au.roundFiguresToString(
deconvolvedMajor * deconvolvedMinor / (synthBeam**2), 2))
writeOut(f, line)
line = " angular separation of phasecal to checksource = %s degree" % (
au.roundFiguresToString(separation, 3))
writeOut(f, line)
if (nScienceFields > 0):
if (nScienceFields > 1):
modifier = 'first'
else:
modifier = 'only'
line = " angular separation of phasecal to %s science field (%d) = %s degree" % (
modifier, scienceid,
au.roundFiguresToString(separation_pcal_science, 3))
writeOut(f, line)
line = " angular separation of checksource to %s science field (%d) = %s degree" % (
modifier, scienceid,
au.roundFiguresToString(separation_check_science, 3))
writeOut(f, line)
f.close()
# end 'for' loop over vislist
return textfiles, pnglist
def lumberjack(vis, spw, field, secsour, stddevfact):
presdir = os.path.realpath('.')
#========== PART 0 ============================================#
#---- USER INPUT AND GENERATE THE SPECTRUM TO ANALYSE ---------#
casalog.origin('lumberjack')
#--- USER INPUT PARAMS
msname = presdir + '/' + vis
SPW = int(spw)
targ = field
#--- set StdDevFactor to default unless otherwise specified
if stddevfact == '':
StdDevFactor = 1.5
else:
StdDevFactor = stddevfact
#print 'Aaaaaaaa'+str(StdDevFactor)
#--- Switch to turn on multiple source extraction if User defines secondarySourcesFile
#--- True to use a user defined list of secondary source positions, False to find spectrum at peak pixel.
if secsour == '':
useSecSources = False
else:
useSecSources = True
if useSecSources:
secondaryFile = secsour
#--- 0.5) Take listobs of an MS and recover the useful information.
listobsFile = msname + '.listobs'
try:
testList = open(listobsFile, 'r')
except IOError:
print "\n >>> Listobs failed to open, making a new one."
listobs(msname, listfile=listobsFile)
testList = open(listobsFile, 'r')
testList.close()
#-- 1) Look into the MS and get useful info
nAnt, antDiam, BW, tInt, sourceFields, chWid, targIdx, numChan = measSetInfo(
msname, SPW, myfield=targ)
Tsys = getTsys(msname, SPW)
#print Tsys
theoRMS = calcSens(nAnt, chWid, tInt, Tsys, antDiam)
print "\n >>> Assuming " + str(nAnt) + " antennas of diameter " + str(
antDiam) + " and channel width of " + str(
chWid) + "\n >>> observed for " + str(tInt) + " sec"
print "\n >>> The theoretical RMS in a single channel is (assuming no flagging etc) is " + str(
theoRMS) + "Jy"
print sourceFields
#-- 1.5) Depending on if a user supplies a SecSource list allow ability to loop around detected continuum sources.
if useSecSources:
print "\n >>> In Secondary Source mode: Using user defined positions and source properties to generate spectra"
secondarySources = np.genfromtxt(
secondaryFile,
dtype=None,
names=['souNum', 'secRA', 'secDec', 'secBmaj', 'secBmin', 'secPA'])
#- How many sources?
try:
useRange = len(secondarySources['souNum'])
except TypeError:
useRange = 1
else:
print "\n >>> In Unknown Source mode: Using peak pixel position to generate spectra"
useRange = 1
for x in range(useRange):
if useRange == 1:
if useSecSources:
source_no = re.split('source',
str(secondarySources['souNum']))[1]
position = secondarySources['secRA'] + '\t' + secondarySources[
'secDec']
thisBmaj = str(secondarySources['secBmaj']) + 'arcsec'
thisBmin = str(secondarySources['secBmin']) + 'arcsec'
thisPA = str(secondarySources['secPA']) + 'deg'
print "\n >>>" + source_no + " " + position + " " + thisBmaj + " " + thisBmin + " " + thisPA
else:
source_no = 'X'
else:
source_no = re.split('source', secondarySources['souNum'][x])[1]
position = secondarySources['secRA'][x] + '\t' + secondarySources[
'secDec'][x]
thisBmaj = str(secondarySources['secBmaj'][x]) + 'arcsec'
thisBmin = str(secondarySources['secBmin'][x]) + 'arcsec'
thisPA = str(secondarySources['secPA'][x]) + 'deg'
print "\n >>>" + source_no + " " + position + " " + thisBmaj + " " + thisBmin + " " + thisPA
#-- 2) Load associated spectrum / or generate one from image !
#-- See if an image exists
imageFile = msname + '_' + targ + '_SPW_' + str(SPW) + '_LumberJack.im'
print "\n >>> looking for " + imageFile
if len(glob.glob(imageFile + '.image')):
useImage = glob.glob(imageFile + '.image')[0]
print "\n >>> Image found, using " + useImage
#--- See if spectrum exists
sourName = re.split('/', useImage)[-1]
sourName = '/' + sourName
specFile = cwd + sourName + '_spec.txt'
if useSecSources:
specFile = getSourcePos2(
useImage, position, cwd, source_no, thisBmaj, thisBmin,
thisPA) # get specta at defined user positions
else:
specFile = getSourcePos(
useImage, cwd
) # get spectrum as peak pixel position (within synth beam)
else:
print "\n >>> No image found... I guess I'll have to make one"
print " >>> Assuming the lowest target field ID is the mosaic centre... NEED BETTER MESSAGE HERE"
#1) Get cell size and field of view
cellSize, FoV = imCellSize(msname, SPW)
imageSize = cleanhelper.getOptimumSize(int((FoV / cellSize) * 2.0))
useImage = imageFile
tclean(
vis=msname,
imagename=useImage,
field=str(targIdx),
spw=str(SPW),
weighting='briggs',
outframe='TOPO',
#phasecenter=str(targIdx),
robust=0.5,
cell=[str(cellSize) + 'arcsec'],
imsize=[imageSize, imageSize], #covers about the FoV
gridder='mosaic',
width=1,
specmode='cube',
nchan=-1,
start='',
niter=0,
restoringbeam='common')
if useSecSources:
specFile = getSourcePos2(
useImage + '.image', position, cwd, source_no, thisBmaj,
thisBmin, thisPA) # get specta at defined user positions
else:
specFile = getSourcePos(
useImage + '.image', cwd
) # get spectrum as peak pixel position (within synth beam)
freq, S = loadFromText(specFile)
#-- 2.5) Test plots
testPlots(freq, S, 'black')
#--- 2.75) Tidy up the data (i.e. remove absorption , weird end chans etc etc)
#--- and obvisous spectral lines
#--- get brightline emission
psd_rms, poss_lines, scale_limits = specFit(S)
poss_lines = poss_lines * 0.0 #-- THIS IS NOT USED SO SET TO ZERO
#--- define where lines aren't
poss_not_lines = (1.0 - poss_lines
) #*(np.max(S)*1.02)#last bit jsut for scaling
lineless_S = poss_not_lines * S
testPlots(freq, poss_not_lines * (np.max(S) * 0.5), 'c')
#-- 3) Sigma clipping loop... explained in calcFuncs.py
freqS, newS, meanS, stdS = sigmaClipperSTD(freq, S, 2.0, 95.5)
testPlots(freqS[np.where(newS != 0)], newS[np.where(newS != 0)],
'orange', '.', 'none')
#-- 4) Testing the gradient approach
gradFreq, gradS = calcGrad(freq, S)
highFreq, highCh, highS = whereHighGrad(gradFreq, gradS, 2.0 * theoRMS)
gradyS = newS[highCh]
gradyS = gradyS[np.where(gradyS != 0.0)]
#-- 6) Gaussian tests
fig3 = plt.figure(3)
ax3 = fig3.add_subplot(111)
#-- Get interquartile range --#
binX = newS[np.where(newS != 0.0)]
q75, q25 = np.percentile(binX, [75, 25])
iqr = q75 - q25
bin_widths = 2.0 * iqr * (len(binX)**(-1.0 / 3.0))
num_bins = (np.max(binX) - np.min(binX)) / bin_widths
valsSC, binsCent = np.histogram(newS[np.where(newS != 0.0)],
bins=np.ceil(num_bins))
valsGR, binsCent = np.histogram(gradyS, bins=binsCent)
centreBins = (binsCent[:-1] + binsCent[1:]) / 2
#--- remove outliers
ax3.hist(newS[np.where(newS != 0.0)],
binsCent,
facecolor='orange',
edgecolor='orange',
alpha=0.15)
ax3.hist(gradyS,
binsCent,
facecolor='cyan',
edgecolor='cyan',
alpha=0.15)
#
ax3.plot(centreBins, valsSC, 'rd--')
ax3.plot(centreBins, valsGR, 'bd--')
#-- Fit the gaussian
meanSC = sum(centreBins * valsSC) / sum(valsSC)
sigmaSC = np.sqrt(sum(valsSC * (centreBins - meanSC)**2) / sum(valsSC))
meanGR = sum(centreBins * valsGR) / sum(valsGR)
sigmaGR = np.sqrt(sum(valsGR * (centreBins - meanGR)**2) / sum(valsGR))
# print 'theoRMs',theoRMS,'!!!'
try:
poptSC, pcovSC = curve_fit(gauss,
centreBins,
valsSC,
p0=[np.max(valsSC), meanSC, sigmaSC])
poptGR, pcovGR = curve_fit(gauss,
centreBins,
valsGR,
p0=[np.max(valsGR), meanGR, sigmaGR])
ax3.plot(centreBins,
gauss(centreBins, poptSC[0], poptSC[1], poptSC[2]),
'ro:',
label='fit')
ax3.plot(centreBins,
gauss(centreBins, poptGR[0], poptGR[1], poptGR[2]),
'bo:',
label='fit')
ax3.set_xlabel('binned flux')
ax3.set_ylabel('Counts')
plt.savefig(targ + '_sourceNo_' + source_no + '_SPW_' + str(SPW) +
'_gaussPlot.png')
ax3.cla()
fig3.clf()
except RuntimeError:
print "\n >>> Curve_fit failed."
#--- Now get the standard deviation and list chans with values within 1,2 and 3 standard deviations.
print " \n >>> Sigma clipper gives... Mean: " + str(poptSC[1])
print " >>> Std. Dev: " + str(poptSC[2])
print " >>> --------"
print " \n >>> Gradient test gives... Mean: " + str(poptGR[1])
print " >>> Std. Dev: " + str(poptGR[2])
print " >>> --------"
#--- SD
testPlots(
freq[np.where(
np.logical_and(S >= (poptSC[1] - StdDevFactor * poptSC[2]),
S <= (poptSC[1] + StdDevFactor * poptSC[2])))],
S[np.where(
np.logical_and(S >= (poptSC[1] - StdDevFactor * poptSC[2]),
S <= (poptSC[1] + StdDevFactor * poptSC[2])))],
'magenta', '.', 'none')
testPlots([np.min(freq), np.max(freq)],
[(poptSC[1] - StdDevFactor * poptSC[2]),
poptSC[1] - StdDevFactor * poptSC[2]], 'magenta')
testPlots([np.min(freq), np.max(freq)],
[(poptSC[1] + StdDevFactor * poptSC[2]),
poptSC[1] + StdDevFactor * poptSC[2]], 'magenta')
testPlots(
freq[np.where(
np.logical_and(S >= (poptGR[1] - StdDevFactor * poptGR[2]),
S <=
(poptGR[1] + StdDevFactor * poptGR[2])))][0],
S[np.where(
np.logical_and(S >= (poptGR[1] - StdDevFactor * poptGR[2]),
S <=
(poptGR[1] + StdDevFactor * poptGR[2])))][0],
'lime', 'x', 'none')
testPlots([np.min(freq), np.max(freq)],
[(poptGR[1] - StdDevFactor * poptGR[2]),
poptGR[1] - StdDevFactor * poptGR[2]], 'lime')
testPlots([np.min(freq), np.max(freq)],
[(poptGR[1] + StdDevFactor * poptGR[2]),
poptGR[1] + StdDevFactor * poptGR[2]], 'lime')
#--- ZERO LINE
testPlots([np.min(freq) - 0.01e9,
np.max(freq) + 0.1e9], [0, 0], 'green')
plt.savefig(targ + '_sourceNo_' + source_no + '_SPW_' + str(SPW) +
'_lineFree.png')
plt.cla()
plt.clf()
#--- Channels to use
print " \n >>> Channels in the " + str(
StdDevFactor) + " sigma range of Sigma clip:\n" + str(
np.where(
np.logical_and(S >=
(poptSC[1] - StdDevFactor * poptSC[2]), S <=
(poptSC[1] + StdDevFactor * poptSC[2])))[0])
print " \n >>> Channels in the " + str(
StdDevFactor) + " sigma range of Gradient test:\n" + str(
np.where(
np.logical_and(S >=
(poptGR[1] - StdDevFactor * poptGR[2]), S <=
(poptSC[1] + StdDevFactor * poptGR[2])))[0])
sigChans = np.where(
np.logical_and(S >= (poptSC[1] - StdDevFactor * poptSC[2]), S <=
(poptSC[1] + StdDevFactor * poptSC[2])))[0]
gradChans = np.where(
np.logical_and(S >= (poptGR[1] - StdDevFactor * poptGR[2]), S <=
(poptSC[1] + StdDevFactor * poptGR[2])))[0]
#--- Combine the channels from the two tests
combinedChans = np.intersect1d(sigChans, gradChans)
#--- And exclude where specFit thinks lines are... jsut to be safe
if np.sum(
poss_not_lines) == 0.0: #To catch when specFit finds no lines
useChans = combinedChans
else:
useChans = np.intersect1d(((np.where(poss_not_lines == 1.0)[0])),
combinedChans)
# print useChans
spwString = numpyToSPWString(SPW, useChans)
outSPWString = open(
targ + '_sourceNo_' + source_no + '_SPW_' + str(SPW) +
'_LineFreeChans.txt', 'w')
print >> outSPWString, spwString
outSPWString.close()
#========== PART 2 ============================================#
#---- GET ALL THE LINE FREE FILES CLEAR THE UNQIUE SET --------#
useThis = np.zeros(numChan)
for x in range(useRange):
if useRange == 1:
if useSecSources:
source_no = re.split('source',
str(secondarySources['souNum']))[1]
else:
source_no = 'X'
else:
source_no = re.split('source', secondarySources['souNum'][x])[1]
thisSPWstr = open(
targ + '_sourceNo_' + source_no + '_SPW_' + str(SPW) +
'_LineFreeChans.txt', 'r')
for line in thisSPWstr:
nuline = re.sub(str(SPW) + ':', '', line)
nuline = re.sub('\n', '', nuline)
nuline2 = re.split(';', nuline)
for chan in nuline2:
ch = int(chan)
useThis[ch] += 1
thisSPWstr.close()
smoothUseThis = smoothLineFree(useThis, useRange, numChan)
allSourceUseChans = np.where(smoothUseThis == useRange)[0]
allSourceSpwString = numpyToSPWString(SPW, allSourceUseChans)
allSourceSpwStringChunk = chunkChansSPWformat(allSourceSpwString)
if useSecSources:
allSourceOutSPW = open(
targ + '_allSource_SPW_' + str(SPW) + '_LineFreeChans.txt', 'w')
else:
allSourceOutSPW = open(
targ + '_SourceX_SPW_' + str(SPW) + '_LineFreeChans.txt', 'w')
print >> allSourceOutSPW, allSourceSpwStringChunk
allSourceOutSPW.close()
def ksc_sim_gauss(projname='sim_7m_array_1GHz_gaus',
antennalist='ksc-7m.cfg',
dishdiam=7,
imagename=None,
indirection='J2000 14h26m46.0s -14d31m22.0s',
incell='1arcsec',
frequency='1.0GHz',
inwidth='1MHz',
radec_offset=[100., 100.],
flux=50.,
majoraxis='40arcsec',
minoraxis='27arcsec',
positionangle='45.0deg',
imsize=[512, 512]):
"""
Simulate observation of an input Gaussian model
:param projname: project name for simobserve
:param antennalist: cfg file of the array configuration
:param dishdiam: diameter of each dish, in meters
:param imagename: name (and path) for output clean image, psf, etc.
:param indirection: phase center of the observation
:param incell: pixel scale of the model/simulated image
:param frequency: central frequency
:param inwidth: frequency bandwidth
:param radec_offset: offset of the Gaussian source from the phasecenter, in arcsec
:param flux: total flux of the Gaussian source, in solar flux unit (sfu)
:param majoraxis: FWHM size of the Gaussian source along the major axis
:param minoraxis: FWHM size of the Gaussian source along the minor axis
:param positionangle: position angle of the Gaussian source
:param imsize: size of the model/simulated image, in pixels (x and y)
:return:
"""
# set voltage patterns and primary beams for KSC 7 m. This is a placeholder for now (but required for PB correction)
vp = vptool()
if len(vp.getvp(telescope='KSC').keys()) == 0:
vprec = vp.setpbairy(telescope='KSC',
dishdiam='{0:.1f}m'.format(dishdiam),
blockagediam='0.75m',
maxrad='1.784deg',
reffreq='1.0GHz',
dopb=True)
# make a Gaussian source
cl = cltool()
ia = iatool()
cl.addcomponent(dir=indirection,
flux=flux * 1e4,
fluxunit='Jy',
freq=frequency,
shape="Gaussian",
majoraxis=majoraxis,
minoraxis=minoraxis,
positionangle=positionangle)
ia.fromshape("Gaussian.im", imsize + [1, 1], overwrite=True)
cs = ia.coordsys()
cs.setunits(['rad', 'rad', '', 'Hz'])
cell_rad = qa.convert(qa.quantity(incell), "rad")['value']
cs.setincrement([-cell_rad, cell_rad], 'direction')
ra_ref = qa.toangle(indirection.split(' ')[1])
dec_ref = qa.toangle(indirection.split(' ')[2])
ra = ra_ref['value'] - radec_offset[0] / 3600.
dec = dec_ref['value'] - radec_offset[1] / 3600.
cs.setreferencevalue([
qa.convert('{0:.4f}deg'.format(ra), 'rad')['value'],
qa.convert('{0:.4f}deg'.format(dec), 'rad')['value']
],
type="direction")
cs.setreferencevalue("1.0GHz", 'spectral')
cs.setincrement('10MHz', 'spectral')
ia.setcoordsys(cs.torecord())
ia.setbrightnessunit("Jy/pixel")
ia.modify(cl.torecord(), subtract=False)
ia.close()
simobserve(project=projname,
skymodel='Gaussian.im',
indirection=indirection,
incell=incell,
incenter=frequency,
inwidth=inwidth,
hourangle='transit',
refdate='2014/11/01',
totaltime='120s',
antennalist=antennalist,
obsmode='int',
overwrite=True)
if not imagename:
imagename = projname + '/tst'
tclean(vis=projname + '/' + projname + '.' + antennalist.split('.')[0] +
'.ms',
imagename=imagename,
imsize=imsize,
cell=incell,
phasecenter=indirection,
niter=200,
interactive=False)
viewer(projname + '/' + projname + '.' + antennalist.split('.')[0] +
'.skymodel')
viewer(imagename + '.psf')
viewer(imagename + '.image')
# re-clean a wider area now
full_imagename = (imagename +
"_{field}_r{robust}_allcont_clean1e4_{threshold}".format(
field=selfcal_fields[0].replace(" ", "_"),
robust=0.5,
threshold='0.5mJy',
))
os.system('rm -r {0}.mask'.format(full_imagename))
tclean(
vis=cont_vis,
field=selfcal_fields[0],
spw='',
imsize=2000,
phasecenter='J2000 17h47m20.166 -28d23m04.968',
cell='0.01arcsec',
imagename=full_imagename,
robust=0.5,
nterms=2,
deconvolver='mtmfs',
specmode='mfs',
gridder=gridder,
niter=10000,
threshold='3.5mJy',
scales=[0, 3, 9, 27],
savemodel='none',
mask='',
)
makefits(full_imagename, cleanup=False)
)
imagename = 'NH322_zoom_on_SgrB2M_40to80kms'
if not os.path.exists(imagename + ".image.pbcor.fits"):
tclean(
vis=merged_ms,
imagename=imagename,
imsize=500,
cell='0.02arcsec',
field=field,
savemodel='modelcolumn',
restfreq=restfreq,
nchan=50, # 0.8 km/s channels
start='40km/s',
phasecenter='J2000 17h47m20.178 -28d23m04.109',
niter=10000,
threshold='50mJy',
gridder='standard',
deconvolver='hogbom',
specmode='cube',
weighting='briggs',
pblimit=0.2,
interactive=False,
outframe='LSRK',
datacolumn='data',
robust=0.5,
)
makefits(imagename)
caltable1 = 'NH322_selfcal_iter1_phase30s.cal'
gaincal(
# create a dirty image for masking
cleanbox_mask_image = 'cleanbox_mask_{0}.image'.format(field_nospace)
imagename = '{0}_QbandAarray_cont_spws_continuum_cal_dirty_2terms_robust0'.format(
field_nospace)
if not os.path.exists('{0}.image.tt0.pbcor.fits'.format(imagename)):
tclean(
vis=selfcal_vis,
imagename=imagename,
# use all fields # field=field,
field="Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q,Sgr B2 S Q",
spw='',
weighting='briggs',
robust=0.0,
phasecenter=phasecenter[field],
imsize=imsize[field],
cell=['0.01 arcsec'],
threshold='1 Jy',
niter=0,
gridder='standard',
specmode='mfs',
deconvolver='mtmfs',
outframe='LSRK',
savemodel='none',
scales=[0, 3, 9, 27],
nterms=2,
selectdata=True,
)
makefits(imagename)
if not os.path.exists('cleanbox_mask_{0}.fits'.format(field_nospace)):
dirtyimage = imagename + '.image.tt0.pbcor'
def myclean(vis,
name,
linename,
spws,
imsize=2000,
cell='0.04arcsec',
fields=[
"Sgr B2 NM Q",
"Sgr B2 MS Q",
"Sgr B2 N Q",
],
phasecenters=None,
niter=1000,
threshold='25mJy',
robust=0.5,
savemodel='none',
overwrite=False,
**kwargs):
if hasattr(spws, 'items'):
assert not isinstance(vis, str)
spws = [spws[k] for k in vis]
for field in fields:
if phasecenters is not None:
phasecenter = phasecenters[field]
else:
phasecenter = ''
imagename = (
"{name}_{field}_r{robust}_{linename}_clean1e4_{threshold}".format(
name=name,
field=field.replace(" ", "_"),
robust=robust,
threshold=threshold,
linename=linename,
))
if os.path.exists(imagename + ".image.pbcor.fits") and not overwrite:
print("Skipping {0} because it's done".format(imagename))
continue
tclean(vis=vis,
field=field,
spw=spws,
imsize=[imsize, imsize],
cell=cell,
imagename=imagename,
niter=niter,
threshold=threshold,
robust=robust,
gridder='standard',
deconvolver='hogbom',
specmode='cube',
weighting='briggs',
pblimit=0.2,
interactive=False,
outframe='LSRK',
datacolumn='corrected',
savemodel=savemodel,
phasecenter=phasecenter,
**kwargs)
makefits(imagename)
imagename = '18A-229_Q_mosaic_selfcal_iter0_dirty'
if not os.path.exists(imagename+".image.tt0.pbcor"):
# do a full-mosaic clean to enable mask creation
tclean(
vis=cont_vis,
spw='',
field="Sgr B2 NM Q,Sgr B2 MS Q,Sgr B2 S Q",
phasecenter='J2000 17h47m19.693 -28d23m11.527',
imsize=[9000,9000],
cell='0.02arcsec',
imagename=imagename,
niter=0,
threshold='1000mJy',
robust=0.5,
gridder='mosaic',
scales=[0,3,9],
deconvolver='mtmfs',
specmode='mfs',
nterms=2,
weighting='briggs',
pblimit=0.2,
interactive=False,
outframe='LSRK',
savemodel='none',
)
makefits(imagename)
cleanbox_mask = 'cleanbox_mask.mask'
cleanbox_mask_image = 'cleanbox_mask_SgrB2.image'
def myclean(
vis,
name,
spws="2,4,5,6,7,8,9,11,12,13,14,15,16,17,18,21,22,23,24,25,27,28,29,30,31,33,34,35,36,38,41,43,44,46,47,48,49,51,52,53,54,55,56,57,58,59,60,62,63,64",
imsize=8000,
cell='0.01arcsec',
fields=[
"Sgr B2 N Q",
"Sgr B2 NM Q",
"Sgr B2 MS Q",
"Sgr B2 S Q",
"Sgr B2 DS1 Q",
"Sgr B2 DS2 Q",
"Sgr B2 DS3 Q",
],
niter=10000,
threshold='0.75mJy',
robust=0.5,
savemodel='none',
gridder='standard',
phasecenters=None,
mask='',
scales=[],
datacolumn='corrected',
noneg=True,
cleanup=True,
**kwargs):
for field in fields:
imagename = (
"{name}_{field}_r{robust}_allcont_clean1e4_{threshold}".format(
name=name,
field=field.replace(" ", "_"),
robust=robust,
threshold=threshold))
if phasecenters is not None:
phasecenter = phasecenters[field]
else:
phasecenter = ''
if not os.path.exists(imagename + ".image.tt0.pbcor.fits"):
rslt = tclean(vis=vis,
field=field,
spw=spws,
imsize=[imsize, imsize],
cell=cell,
imagename=imagename,
niter=niter,
threshold=threshold,
phasecenter=phasecenter,
robust=robust,
gridder=gridder,
deconvolver='mtmfs',
specmode='mfs',
nterms=2,
weighting='briggs',
pblimit=0.1,
interactive=False,
outframe='LSRK',
datacolumn=datacolumn,
savemodel=savemodel,
scales=scales,
mask=mask,
**kwargs)
makefits(imagename, cleanup=cleanup)
else:
casalog.post("Skipping {0}".format(imagename), origin='myclean')
if noneg and os.path.exists(imagename + ".model.tt0"):
noneg_model(modelname=imagename + ".model.tt0",
ms=vis,
imagename=imagename,
imsize=[imsize, imsize],
cell=cell,
phasecenter=phasecenter,
gridder=gridder,
robust=robust,
scales=scales,
**kwargs)