def rgb3(r, g, b, jpgname):
""" Convert 3 2D-numpy arrays into a colorful JPG image
It needs the PIL module, which CASA doesn't have
but we would like to try this out one of these moons...
"""
from PIL import Image
# skip all the shape tests and being 2D
sh = r.shape
rgb = np.zeros((sh[0],sh[1],3), 'uint8')
rgb[..., 0] = r*256
rgb[..., 1] = g*256
rgb[..., 2] = b*256
img = Image.fromarray(rgb)
img.save(jpgname)
def run(self):
"""Runs the task.
Parameters
----------
None
Returns
-------
None
"""
self._summary = {}
pvslicesummary = []
sumslicetype = 'slice'
sliceargs = []
dt = utils.Dtime("PVSlice")
# import here, otherwise sphinx cannot parse
from impv import impv
from imsmooth import imsmooth
pvslice = self.getkey('slice') # x_s,y_s,x_e,y_e (start and end of line)
pvslit = self.getkey('slit') # x_c,y_c,len,pa (center, length and PA of line)
# BDP's used :
# b10 = input BDP
# b11 = input BDP (moment)
# b12 = input BDP (new style cubestats w/ maxpos)
# b2 = output BDP
b10 = self._bdp_in[0] # input SpwCube
fin = b10.getimagefile(bt.CASA) # input name
b11 = self._bdp_in[1] #
b12 = self._bdp_in[2]
clip = self.getkey('clip') # clipping to data for Moment-of-Inertia
gamma = self.getkey('gamma') # gamma factor to data for Moment-of-Inertia
if b11 != None and len(pvslice) == 0 and len(pvslit) == 0:
# if a map (e.g. cubesum ) given, and no slice/slit, get a best pvslice from that
(pvslice,clip) = map_to_slit(self.dir(b11.getimagefile(bt.CASA)),clip=clip,gamma=gamma)
elif b12 != None and len(pvslice) == 0 and len(pvslit) == 0:
# PPP doesn't seem to work too well yet
logging.debug("testing new slice computation from a PPP")
max = b12.table.getColumnByName("max")
maxposx = b12.table.getColumnByName("maxposx")
maxposy = b12.table.getColumnByName("maxposy")
if maxposx == None:
raise Exception,"PPP was not enabled in your CubeStats"
(pvslice,clip) = tab_to_slit([maxposx,maxposy,max],clip=clip,gamma=gamma)
sliceargs = deepcopy(pvslice)
if len(sliceargs)==0:
logging.warning("no slice for plot yet")
# ugh, this puts single quotes around the numbers
formattedslice = str(["%.2f" % a for a in sliceargs])
taskargs = "slice="+formattedslice
dt.tag("slice")
pvname = self.mkext(fin,'pv') # output image name
b2 = PVSlice_BDP(pvname)
self.addoutput(b2)
width = self.getkey('width') # @todo also: "4arcsec" (can't work since it's a single keyword)
if len(pvslice) == 4:
start = pvslice[:2] # @todo also allow: ["14h20m20.5s","-30d45m25.4s"]
end = pvslice[2:]
impv(self.dir(fin), self.dir(pvname),"coords",start=start,end=end,width=width,overwrite=True)
elif len(pvslit) == 4:
sumslicetype = 'slit'
sliceargs = deepcopy(pvslit)
formattedslice = str(["%.2f" % a for a in sliceargs])
taskargs = "slit="+formattedslice
# length="40arcsec" same as {"value": 40, "unit": "arcsec"})
center = pvslit[:2] # @todo also: ["14h20m20.5s","-30d45m25.4s"].
length = pvslit[2] # @todo also: "40arcsec", {"value": 40, "unit": "arcsec"})
if type(pvslit[3]) is float or type(pvslit[3]) is int:
pa = "%gdeg" % pvslit[3]
else:
pa = pvslit[3]
impv(self.dir(fin), self.dir(pvname),"length",center=center,length=length,pa=pa,width=width,overwrite=True)
else:
raise Exception,"no valid input slit= or slice= or bad Moment_BDP input"
sliceargs.append(width)
taskargs = taskargs + " width=%d" % width
dt.tag("impv")
smooth = self.getkey('pvsmooth')
if len(smooth) > 0:
if len(smooth) == 1:
smooth.append(smooth[0])
major = '%dpix' % smooth[0]
minor = '%dpix' % smooth[1]
logging.info("imsmooth PV slice: %s %s" % (major,minor))
imsmooth(self.dir(pvname), outfile=self.dir(pvname)+'.smooth',kernel='boxcar',major=major,minor=minor)
dt.tag("imsmooth")
# utils.rename(self.dir(pvname)+'.smooth',self.dir(pvname))
# @todo we will keep the smooth PVslice for inspection, no further flow work
# get some statistics
data = casautil.getdata_raw(self.dir(pvname))
rpix = stats.robust(data.flatten())
r_mean = rpix.mean()
r_std = rpix.std()
r_max = rpix.max()
logging.info("PV stats: mean/std/max %f %f %f" % (r_mean, r_std, r_max))
logging.regression("PVSLICE: %f %f %f" % (r_mean, r_std, r_max))
myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir())
# hack to get a slice on a mom0map
# @todo if pmode is not png, can viewer handle this?
figname = pvname + ".png"
slicename = self.dir(figname)
overlay = "pvoverlay"
if b11 != None:
f11 = b11.getimagefile(bt.CASA)
taskinit.tb.open(self.dir(f11))
data = taskinit.tb.getcol('map')
nx = data.shape[0]
ny = data.shape[1]
taskinit.tb.close()
d1 = np.flipud(np.rot90 (data.reshape((nx,ny))))
if len(pvslice) == 4:
segm = [[pvslice[0],pvslice[2],pvslice[1],pvslice[3]]]
pa = np.arctan2(pvslice[2]-pvslice[0],pvslice[1]-pvslice[3])*180.0/np.pi
title = "PV Slice location : slice PA=%.1f" % pa
elif len(pvslit) == 4:
# can only do this now if using pixel coordinates
xcen = pvslit[0]
ycen = ny-pvslit[1]-1
slen = pvslit[2]
pard = pvslit[3]*np.pi/180.0
cosp = np.cos(pard)
sinp = np.sin(pard)
halflen = 0.5*slen
segm = [[xcen-halflen*sinp,xcen+halflen*sinp,ycen-halflen*cosp,ycen+halflen*cosp]]
pa = pvslit[3]
title = "PV Slice location : slit PA=%g" % pa
else:
# bogus, some error in pvslice
logging.warning("bogus segm since pvslice=%s" % str(pvslice))
segm = [[10,20,10,20]]
pa = -999.999
title = "PV Slice location - bad PA"
logging.info("MAP1 segm %s %s" % (str(segm),str(pvslice)))
if d1.max() < clip:
logging.warning("datamax=%g, clip=%g" % (d1.max(), clip))
title = title + ' (no signal over %g?)' % clip
myplot.map1(d1,title,overlay,segments=segm,thumbnail=True)
else:
myplot.map1(d1,title,overlay,segments=segm,range=[clip],thumbnail=True)
dt.tag("plot")
overlayname = myplot.getFigure(figno=myplot.figno,relative=True)
overlaythumbname = myplot.getThumbnail(figno=myplot.figno,relative=True)
Qover = True
else:
Qover = False
implot = ImPlot(pmode=self._plot_mode,ptype=self._plot_type,abspath=self.dir())
implot.plotter(rasterfile=pvname, figname=pvname, colorwedge=True)
thumbname = implot.getThumbnail(figno=implot.figno,relative=True)
figname = implot.getFigure(figno=implot.figno,relative=True)
if False:
# debug:
#
# @todo tmp1 is ok, tmp2 is not displaying the whole thing
# old style: viewer() seems to plot full image, but imview() wants square pixels?
casa.viewer(infile=self.dir(pvname), outfile=self.dir('tmp1.pv.png'), gui=False, outformat="png")
casa.imview(raster={'file':self.dir(pvname), 'colorwedge' : True, 'scaling':-1},
axes={'y':'Declination'},
out=self.dir('tmp2.pv.png'))
#
# -> this one works, axes= should be correct
# imview(raster={'file':'x.pv', 'colorwedge' : True, 'scaling':-1},axes={'y':'Frequency'})
#
# @TODO big fixme, we're going to reuse 'tmp1.pv.png' because implot give a broken view
figname = 'tmp1.pv.png'
# @todo technically we don't know what map it was overlay'd on.... CubeSum/Moment0
overlaycaption = "Location of position-velocity slice overlaid on a CubeSum map"
pvcaption = "Position-velocity diagram through emission centroid"
pvimage = Image(images={bt.CASA : pvname, bt.PNG : figname},thumbnail=thumbname,thumbnailtype=bt.PNG, description=pvcaption)
b2.setkey("image",pvimage)
b2.setkey("mean",float(r_mean))
b2.setkey("sigma",float(r_std))
if Qover:
thispvsummary = [sumslicetype,sliceargs,figname,thumbname,pvcaption,overlayname,overlaythumbname,overlaycaption,pvname,fin]
else:
thispvsummary = [sumslicetype,sliceargs,figname,thumbname,pvcaption,pvname,fin]
# Yes, this is a nested list. Against the day when PVSLICE can
# compute multiple slices per map.
pvslicesummary.append(thispvsummary)
self._summary["pvslices"] = SummaryEntry(pvslicesummary,"PVSlice_AT",self.id(True),taskargs)
dt.tag("done")
dt.end()
def run(self):
dt = utils.Dtime("PVCorr")
self._summary = {}
numsigma = self.getkey("numsigma")
mode = 1 # PV corr mode (1,2,3)
normalize = True
# normalize = False
b1 = self._bdp_in[0] # PVSlice_BDP
fin = b1.getimagefile(bt.CASA) # CASA image
data = casautil.getdata_raw(
self.dir(fin)) # grab the data as a numpy array
self.myplot = APlot(ptype=self._plot_type,
pmode=self._plot_mode,
abspath=self.dir())
#print 'DATA[0,0]:',data[0,0]
#print 'pv shape: ',data.shape
npos = data.shape[0]
nvel = data.shape[1]
dt.tag("getdata")
b2 = self._bdp_in[1] # CubeStats_BDP
sigma = b2.sigma # global sigma in the cube
cutoff = numsigma * sigma
freq = b2.table.getColumnByName("frequency")
chans = self.getkey("range") # range of channels, if used
if len(chans) > 0:
if len(chans) != 2:
logging.fatal("range=%s" % chans)
raise Exception, "range= needs two values, left and right (inclusive) channel"
ch0 = chans[0]
ch1 = chans[1]
else:
nchan = self.getkey("nchan")
imstat0 = casa.imstat(self.dir(fin)) # @todo can use data[] now
xmaxpos = imstat0['maxpos'][0]
ymaxpos = imstat0['maxpos'][1]
logging.info("MAXPOS-VEL %s %g" %
(str(imstat0['maxpos']), imstat0['max'][0]))
if nchan > 0:
# expand around it, later ch0,ch1 will be checked for running off the edge
ch0 = ymaxpos - nchan / 2
ch1 = ymaxpos + nchan / 2
else:
# watershed down to find ch0 and ch1 ?
# this doesn't work well in crowded areas
ch0 = ymaxpos
ch1 = ymaxpos
spmax = data.max(axis=0)
k = spmax.argmax()
n = len(spmax)
logging.debug('spmax %s %d %g' %
(str(spmax.shape), k, spmax[k]))
# find lower cutoff
for i in range(n):
ch0 = ymaxpos - i
if ch0 < 0: break
if spmax[ch0] < cutoff: break
ch0 = ch0 + 1
# find higher cutoff
for i in range(n):
ch1 = ymaxpos + i
if ch1 == n: break
if spmax[ch1] < cutoff: break
ch1 = ch1 - 1
dt.tag("imstat")
bdp_name = self.mkext(fin, "pvc") # output PVCorr_BDP
b3 = PVCorr_BDP(bdp_name)
self.addoutput(b3)
if ch0 < 0 or ch1 >= nvel:
# this probably only happens to small cubes (problematic for PVCorr)
# or when the strongest line is really close to the edge of the band
# (which is probably ok)
if ch0 < 0 and ch1 >= nvel:
logging.warning("Serious issues with the size of this cube")
if ch0 < 0:
logging.warning("Resetting ch0 edge to 0")
ch0 = 0
if ch1 >= nvel:
ch1 = nvel - 1
logging.warning("Resetting ch1 edge to the maximum")
if ch0 > ch1:
logging.warning("Sanity swapping ch0,1 due to likely noisy data")
ch0, ch1 = ch1, ch0
if mode == 1:
out, rms = mode1(data, ch0, ch1, cutoff, normalize)
corr = out
elif mode == 2:
out, rms = mode2(data, ch0, ch1, cutoff) # slower 2D version
corr = out[
npos /
2, :] # center cut, but could also try feature detection
elif mode == 3:
out, rms = self.mode3(data, ch0, ch1,
cutoff) # Doug's faster 2D version
# get the peak of each column
corr = np.amax(out, axis=0)
# print "PVCORR SHAPE ",corr.shape," mode", mode
if len(corr) > 0:
# print "SHAPE out:",out.shape,corr.shape,npos/2
ch = range(len(corr))
if len(corr) != len(freq):
logging.fatal("ch (%d) and freq (%d) do not have same size" %
(len(corr), len(freq)))
raise Exception, "ch and freq do not have same dimension"
dt.tag("mode")
labels = ["channel", "frequency", "pvcorr"]
units = ["number", "GHz", "N/A"]
data = (ch, freq, corr)
table = Table(columns=labels,
units=units,
data=np.column_stack(data))
else:
# still construct a table, but with no rows
labels = ["channel", "frequency", "pvcorr"]
units = ["number", "GHz", "N/A"]
table = Table(columns=labels, units=units)
b3.setkey("table", table)
b3.setkey("sigma", float(rms))
dt.tag("table")
if len(corr) > 0:
table.exportTable(self.dir("testPVCorr.tab"),
cols=['frequency', 'pvcorr'])
test_single(ch, freq, corr)
logging.regression("PVC: %f %f" % (corr.min(), corr.max()))
title = 'PVCorr mode=%d [%d,%d] %g' % (mode, ch0, ch1, cutoff)
x = ch
xlab = 'Channel'
y = [corr]
ylab = 'PV Correlation'
p1 = "%s_%d" % (bdp_name, 0)
segp = []
segp.append([0, len(ch), 0.0, 0.0])
segp.append([0, len(ch), 3.0 * rms, 3.0 * rms])
# @todo: in principle we know with given noise and size of box, what the sigma in pvcorr should be
self.myplot.plotter(x,
y,
title,
figname=p1,
xlab=xlab,
ylab=ylab,
segments=segp,
thumbnail=True)
#out1 = np.rot90 (data.reshape((nvel,npos)) )
if mode > 1:
self.myplot.map1(data=out,
title="testing PVCorr_AT: mode%d" % mode,
figname='testPVCorr',
thumbnail=True)
taskargs = "numsigma=%.1f range=[%d,%d]" % (numsigma, ch0, ch1)
caption = "Position-velocity correlation plot"
thumbname = self.myplot.getThumbnail(figno=self.myplot.figno,
relative=True)
figname = self.myplot.getFigure(figno=self.myplot.figno,
relative=True)
image = Image(images={bt.PNG: figname},
thumbnail=thumbname,
thumbnailtype=bt.PNG,
description=caption)
b3.image.addimage(image, "pvcorr")
self._summary["pvcorr"] = SummaryEntry(
[figname, thumbname, caption, fin], "PVCorr_AT", self.id(True),
taskargs)
else:
self._summary["pvcorr"] = None
logging.warning("No summary")
logging.regression("PVC: -1")
dt.tag("done")
dt.end()
def run(self):
"""Runs the task.
Parameters
----------
None
Returns
-------
None
"""
self._summary = {}
dt = utils.Dtime("CubeStats")
#maxvrms = 2.0 # maximum variation in rms allowed (hardcoded for now)
#maxvrms = -1.0 # turn maximum variation in rms allowed off
maxvrms = self.getkey("maxvrms")
psample = -1
psample = self.getkey("psample")
# BDP's used :
# b1 = input BDP
# b2 = output BDP
b1 = self._bdp_in[0]
fin = b1.getimagefile(bt.CASA)
bdp_name = self.mkext(fin,'cst')
b2 = CubeStats_BDP(bdp_name)
self.addoutput(b2)
# PeakPointPlot
use_ppp = self.getkey("ppp")
# peakstats: not enabled for mortal users yet
# peakstats = (psample=1, numsigma=4, minchan=3, maxgap=2, peakfit=False)
pnumsigma = 4
minchan = 3
maxgap = 2
peakfit = False # True will enable a true gaussian fit
# numsigma: adding all signal > numsigma ; not user enabled; for peaksum.
numsigma = -1.0
numsigma = 3.0
# grab the new robust statistics. If this is used, 'rms' will be the RMS,
# else we will use RMS = 1.4826*MAD (MAD does a decent job on outliers as well)
# and was the only method available before CASA 4.4 when robust was implemented
robust = self.getkey("robust")
rargs = casautil.parse_robust(robust)
nrargs = len(rargs)
if nrargs == 0:
sumrargs = "medabsdevmed" # for the summary, indicate the default robust
else:
sumrargs = str(rargs)
self._summary["rmsmethd"] = SummaryEntry([sumrargs,fin],"CubeStats_AT",self.id(True))
#@todo think about using this instead of putting 'fin' in all the SummaryEntry
#self._summary["casaimage"] = SummaryEntry(fin,"CubeStats_AT",self.id(True))
# extra CASA call to get the freq's in GHz, as these are not in imstat1{}
# @todo what if the coordinates are not in FREQ ?
# Note: CAS-7648 bug on 3D cubes
if False:
# csys method
ia.open(self.dir(fin))
csys = ia.coordsys()
spec_axis = csys.findaxisbyname("spectral")
# ieck, we need a valid position, or else it will come back and "Exception: All selected pixels are masked"
#freqs = ia.getprofile(spec_axis, region=rg.box([0,0],[0,0]))['coords']/1e9
#freqs = ia.getprofile(spec_axis)['coords']/1e9
freqs = ia.getprofile(spec_axis,unit="GHz")['coords']
dt.tag("getprofile")
else:
# old imval method
#imval0 = casa.imval(self.dir(fin),box='0,0,0,0') # this fails on 3D
imval0 = casa.imval(self.dir(fin))
freqs = imval0['coords'].transpose()[2]/1e9
dt.tag("imval")
nchan = len(freqs)
chans = np.arange(nchan)
# call CASA to get what we want
# imstat0 is the whole cube, imstat1 the plane based statistics
# warning: certain robust stats (**rargs) on the whole cube are going to be very slow
dt.tag("start")
imstat0 = casa.imstat(self.dir(fin), logfile=self.dir('imstat0.logfile'),append=False,**rargs)
dt.tag("imstat0")
imstat1 = casa.imstat(self.dir(fin),axes=[0,1],logfile=self.dir('imstat1.logfile'),append=False,**rargs)
dt.tag("imstat1")
# imm = casa.immoments(self.dir(fin),axis='spec', moments=8, outfile=self.dir('ppp.im'))
if nrargs > 0:
# need to get the peaks without rubust
imstat10 = casa.imstat(self.dir(fin), logfile=self.dir('imstat0.logfile'),append=True)
dt.tag("imstat10")
imstat11 = casa.imstat(self.dir(fin),axes=[0,1],logfile=self.dir('imstat1.logfile'),append=True)
dt.tag("imstat11")
# grab the relevant plane-based things from imstat1
if nrargs == 0:
mean = imstat1["mean"]
sigma = imstat1["medabsdevmed"]*1.4826 # see also: astropy.stats.median_absolute_deviation()
peakval = imstat1["max"]
minval = imstat1["min"]
else:
mean = imstat1["mean"]
sigma = imstat1["rms"]
peakval = imstat11["max"]
minval = imstat11["min"]
if True:
# work around a bug in imstat(axes=[0,1]) for last channel [CAS-7697]
for i in range(len(sigma)):
if sigma[i] == 0.0:
minval[i] = peakval[i] = 0.0
# too many variations in the RMS ?
sigma_pos = sigma[np.where(sigma>0)]
smin = sigma_pos.min()
smax = sigma_pos.max()
logging.info("sigma varies from %f to %f; %d/%d channels ok" % (smin,smax,len(sigma_pos),len(sigma)))
if maxvrms > 0:
if smax/smin > maxvrms:
cliprms = smin * maxvrms
logging.warning("sigma varies too much, going to clip to %g (%g > %g)" % (cliprms, smax/smin, maxvrms))
sigma = np.where(sigma < cliprms, sigma, cliprms)
# @todo (and check again) for foobar.fits all sigma's became 0 when robust was selected
# was this with mask=True/False?
# PeakPointPlot (can be expensive, hence the option)
if use_ppp:
logging.info("Computing MaxPos for PeakPointPlot")
xpos = np.zeros(nchan)
ypos = np.zeros(nchan)
peaksum = np.zeros(nchan)
ia.open(self.dir(fin))
for i in range(nchan):
if sigma[i] > 0.0:
plane = ia.getchunk(blc=[0,0,i,-1],trc=[-1,-1,i,-1],dropdeg=True)
v = ma.masked_invalid(plane)
v_abs = np.absolute(v)
max = np.unravel_index(v_abs.argmax(), v_abs.shape)
xpos[i] = max[0]
ypos[i] = max[1]
if numsigma > 0.0:
peaksum[i] = ma.masked_less(v,numsigma * sigma[i]).sum()
peaksum = np.nan_to_num(peaksum) # put 0's where nan's are found
ia.close()
dt.tag("ppp")
nzeros = len(np.where(sigma<=0.0))
if nzeros > 0:
zeroch = np.where(sigma<=0.0)
logging.warning("There are %d fully masked channels (%s)" % (nzeros,str(zeroch)))
# construct the admit Table for CubeStats_BDP
# note data needs to be a tuple, later to be column_stack'd
if use_ppp:
labels = ["channel" ,"frequency" ,"mean" ,"sigma" ,"max" ,"maxposx" ,"maxposy" ,"min", "peaksum"]
units = ["number" ,"GHz" ,"Jy/beam" ,"Jy/beam" ,"Jy/beam" ,"number" ,"number" ,"Jy/beam", "Jy"]
data = (chans ,freqs ,mean ,sigma ,peakval ,xpos ,ypos ,minval, peaksum)
else:
labels = ["channel" ,"frequency" ,"mean" ,"sigma" ,"max" ,"min"]
units = ["number" ,"GHz" ,"Jy/beam" ,"Jy/beam" ,"Jy/beam" ,"Jy/beam"]
data = (chans ,freqs ,mean ,sigma ,peakval ,minval)
table = Table(columns=labels,units=units,data=np.column_stack(data))
b2.setkey("table",table)
# get the full cube statistics, it depends if robust was pre-selected
if nrargs == 0:
mean0 = imstat0["mean"][0]
sigma0 = imstat0["medabsdevmed"][0]*1.4826
peak0 = imstat0["max"][0]
b2.setkey("mean" , float(mean0))
b2.setkey("sigma", float(sigma0))
b2.setkey("minval",float(imstat0["min"][0]))
b2.setkey("maxval",float(imstat0["max"][0]))
b2.setkey("minpos",imstat0["minpos"][:3].tolist()) #? [] or array(..dtype=int32) ??
b2.setkey("maxpos",imstat0["maxpos"][:3].tolist()) #? [] or array(..dtype=int32) ??
logging.info("CubeMax: %f @ %s" % (imstat0["max"][0],str(imstat0["maxpos"])))
logging.info("CubeMin: %f @ %s" % (imstat0["min"][0],str(imstat0["minpos"])))
logging.info("CubeRMS: %f" % sigma0)
else:
mean0 = imstat0["mean"][0]
sigma0 = imstat0["rms"][0]
peak0 = imstat10["max"][0]
b2.setkey("mean" , float(mean0))
b2.setkey("sigma", float(sigma0))
b2.setkey("minval",float(imstat10["min"][0]))
b2.setkey("maxval",float(imstat10["max"][0]))
b2.setkey("minpos",imstat10["minpos"][:3].tolist()) #? [] or array(..dtype=int32) ??
b2.setkey("maxpos",imstat10["maxpos"][:3].tolist()) #? [] or array(..dtype=int32) ??
logging.info("CubeMax: %f @ %s" % (imstat10["max"][0],str(imstat10["maxpos"])))
logging.info("CubeMin: %f @ %s" % (imstat10["min"][0],str(imstat10["minpos"])))
logging.info("CubeRMS: %f" % sigma0)
b2.setkey("robust",robust)
rms_ratio = imstat0["rms"][0]/sigma0
logging.info("RMS Sanity check %f" % rms_ratio)
if rms_ratio > 1.5:
logging.warning("RMS sanity check = %f. Either bad sidelobes, lotsa signal, or both" % rms_ratio)
logging.regression("CST: %f %f" % (sigma0, rms_ratio))
# plots: no plots need to be made when nchan=1 for continuum
# however we could make a histogram, overlaying the "best" gauss so
# signal deviations are clear?
logging.info('mean,rms,S/N=%f %f %f' % (mean0,sigma0,peak0/sigma0))
if nchan == 1:
# for a continuum/1-channel we only need to stuff some numbers into the _summary
self._summary["chanrms"] = SummaryEntry([float(sigma0), fin], "CubeStats_AT", self.id(True))
self._summary["dynrange"] = SummaryEntry([float(peak0)/float(sigma0), fin], "CubeStats_AT", self.id(True))
self._summary["datamean"] = SummaryEntry([float(mean0), fin], "CubeStats_AT", self.id(True))
else:
y1 = np.log10(ma.masked_invalid(peakval))
y2 = np.log10(ma.masked_invalid(sigma))
y3 = y1-y2
y4 = np.log10(ma.masked_invalid(-minval))
y5 = y1-y4
y = [y1,y2,y3,y4]
title = 'CubeStats: ' + bdp_name+'_0'
xlab = 'Channel'
ylab = 'log(Peak,Noise,Peak/Noise)'
labels = ['log(peak)','log(rms noise)','log(peak/noise)','log(|minval|)']
myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir())
segp = [[chans[0],chans[nchan-1],math.log10(sigma0),math.log10(sigma0)]]
myplot.plotter(chans,y,title,bdp_name+"_0",xlab=xlab,ylab=ylab,segments=segp,labels=labels,thumbnail=True)
imfile = myplot.getFigure(figno=myplot.figno,relative=True)
thumbfile = myplot.getThumbnail(figno=myplot.figno,relative=True)
image0 = Image(images={bt.PNG:imfile},thumbnail=thumbfile,thumbnailtype=bt.PNG,description="CubeStats_0")
b2.addimage(image0,"im0")
if use_ppp:
# new trial for Lee
title = 'PeakSum: (numsigma=%.1f)' % (numsigma)
ylab = 'Jy*N_ppb'
myplot.plotter(chans,[peaksum],title,bdp_name+"_00",xlab=xlab,ylab=ylab,thumbnail=False)
if True:
# hack ascii table
y30 = np.where(sigma > 0, np.log10(peakval/sigma), 0.0)
table2 = Table(columns=["freq","log(P/N)"],data=np.column_stack((freqs,y30)))
table2.exportTable(self.dir("testCubeStats.tab"))
del table2
# the "box" for the "spectrum" is all pixels. Don't know how to
# get this except via shape.
ia.open(self.dir(fin))
s = ia.summary()
ia.close()
if 'shape' in s:
specbox = (0,0,s['shape'][0],s['shape'][1])
else:
specbox = ()
caption = "Emission characteristics as a function of channel, as derived by CubeStats_AT "
caption += "(cyan: global rms,"
caption += " green: noise per channel,"
caption += " blue: peak value per channel,"
caption += " red: peak/noise per channel)."
self._summary["spectra"] = SummaryEntry([0, 0, str(specbox), 'Channel', imfile, thumbfile , caption, fin], "CubeStats_AT", self.id(True))
self._summary["chanrms"] = SummaryEntry([float(sigma0), fin], "CubeStats_AT", self.id(True))
# @todo Will imstat["max"][0] always be equal to s['datamax']? If not, why not?
if 'datamax' in s:
self._summary["dynrange"] = SummaryEntry([float(s['datamax']/sigma0), fin], "CubeStats_AT", self.id(True))
else:
self._summary["dynrange"] = SummaryEntry([float(imstat0["max"][0]/sigma0), fin], "CubeStats_AT", self.id(True))
self._summary["datamean"] = SummaryEntry([imstat0["mean"][0], fin], "CubeStats_AT", self.id(True))
title = bdp_name + "_1"
xlab = 'log(Peak,Noise,P/N)'
myplot.histogram([y1,y2,y3],title,bdp_name+"_1",xlab=xlab,thumbnail=True)
imfile = myplot.getFigure(figno=myplot.figno,relative=True)
thumbfile = myplot.getThumbnail(figno=myplot.figno,relative=True)
image1 = Image(images={bt.PNG:imfile},thumbnail=thumbfile,thumbnailtype=bt.PNG,description="CubeStats_1")
b2.addimage(image1,"im1")
# note that the 'y2' can have been clipped, which can throw off stats.robust()
# @todo should set a mask for those.
title = bdp_name + "_2"
xlab = 'log(Noise))'
n = len(y2)
ry2 = stats.robust(y2)
y2_mean = ry2.mean()
y2_std = ry2.std()
if n>9: logging.debug("NORMALTEST2: %s" % str(scipy.stats.normaltest(ry2)))
myplot.hisplot(y2,title,bdp_name+"_2",xlab=xlab,gauss=[y2_mean,y2_std],thumbnail=True)
title = bdp_name + "_3"
xlab = 'log(diff[Noise])'
n = len(y2)
# dy2 = y2[0:-2] - y2[1:-1]
dy2 = ma.masked_equal(y2[0:-2] - y2[1:-1],0.0).compressed()
rdy2 = stats.robust(dy2)
dy2_mean = rdy2.mean()
dy2_std = rdy2.std()
if n>9: logging.debug("NORMALTEST3: %s" % str(scipy.stats.normaltest(rdy2)))
myplot.hisplot(dy2,title,bdp_name+"_3",xlab=xlab,gauss=[dy2_mean,dy2_std],thumbnail=True)
title = bdp_name + "_4"
xlab = 'log(Signal/Noise))'
n = len(y3)
ry3 = stats.robust(y3)
y3_mean = ry3.mean()
y3_std = ry3.std()
if n>9: logging.debug("NORMALTEST4: %s" % str(scipy.stats.normaltest(ry3)))
myplot.hisplot(y3,title,bdp_name+"_4",xlab=xlab,gauss=[y3_mean,y3_std],thumbnail=True)
title = bdp_name + "_5"
xlab = 'log(diff[Signal/Noise)])'
n = len(y3)
dy3 = y3[0:-2] - y3[1:-1]
rdy3 = stats.robust(dy3)
dy3_mean = rdy3.mean()
dy3_std = rdy3.std()
if n>9: logging.debug("NORMALTEST5: %s" % str(scipy.stats.normaltest(rdy3)))
myplot.hisplot(dy3,title,bdp_name+"_5",xlab=xlab,gauss=[dy3_mean,dy3_std],thumbnail=True)
title = bdp_name + "_6"
xlab = 'log(Peak+Min)'
n = len(y1)
ry5 = stats.robust(y5)
y5_mean = ry5.mean()
y5_std = ry5.std()
if n>9: logging.debug("NORMALTEST6: %s" % str(scipy.stats.normaltest(ry5)))
myplot.hisplot(y5,title,bdp_name+"_6",xlab=xlab,gauss=[y5_mean,y5_std],thumbnail=True)
logging.debug("LogPeak: m,s= %f %f min/max %f %f" % (y1.mean(),y1.std(),y1.min(),y1.max()))
logging.debug("LogNoise: m,s= %f %f %f %f min/max %f %f" % (y2.mean(),y2.std(),y2_mean,y2_std,y2.min(),y2.max()))
logging.debug("LogDeltaNoise: RMS/sqrt(2)= %f %f " % (dy2.std()/math.sqrt(2),dy2_std/math.sqrt(2)))
logging.debug("LogDeltaP/N: RMS/sqrt(2)= %f %f" % (dy3.std()/math.sqrt(2),dy3_std/math.sqrt(2)))
logging.debug("LogPeak+Min: robust m,s= %f %f" % (y5_mean,y5_std))
# compute two ratios that should both be near 1.0 if noise is 'normal'
ratio = y2.std()/(dy2.std()/math.sqrt(2))
ratio2 = y2_std/(dy2_std/math.sqrt(2))
logging.info("RMS BAD VARIATION RATIO: %f %f" % (ratio,ratio2))
# making PPP plot
if nchan > 1 and use_ppp:
smax = 10
gamma = 0.75
z0 = peakval/peakval.max()
# point sizes
s = np.pi * ( smax * (z0**gamma) )**2
cmds = ["grid", "axis equal"]
title = "Peak Points per channel"
pppimage = bdp_name + '_ppp'
myplot.scatter(xpos,ypos,title=title,figname=pppimage,size=s,color=chans,cmds=cmds,thumbnail=True)
pppimage = myplot.getFigure(figno=myplot.figno,relative=True)
pppthumbnail = myplot.getThumbnail(figno=myplot.figno,relative=True)
caption = "Peak point plot: Locations of per-channel peaks in the image cube " + fin
self._summary["peakpnt"] = SummaryEntry([pppimage, pppthumbnail, caption, fin], "CubeStats_AT", self.id(True))
dt.tag("plotting")
# making PeakStats plot
if nchan > 1 and psample > 0:
logging.info("Computing peakstats")
# grab peak,mean and width values for all peaks
(pval,mval,wval) = peakstats(self.dir(fin),freqs,sigma0,pnumsigma,minchan,maxgap,psample,peakfit)
title = "PeakStats: cutoff = %g" % (sigma0*pnumsigma)
xlab = 'Peak value'
ylab = 'FWHM (channels)'
pppimage = bdp_name + '_peakstats'
cval = mval
myplot.scatter(pval,wval,title=title,xlab=xlab,ylab=ylab,color=cval,figname=pppimage,thumbnail=False)
dt.tag("peakstats")
# myplot.final() # pjt debug
# all done!
dt.tag("done")
taskargs = "robust=" + sumrargs
if use_ppp:
taskargs = taskargs + " ppp=True"
else:
taskargs = taskargs + " ppp=False"
for v in self._summary:
self._summary[v].setTaskArgs(taskargs)
dt.tag("summary")
dt.end()
def run(fileName):
# instantiate the class
a = ad.Admit()
s = spw.SpwCube_BDP(taskid=1234, xmlFile="spw_test_object.bdp")
image = Image({bt.CASA: fileName})
s.image = image
# instantiate a moment AT
m = ma.Moment_AT()
# add the moment AT to the admit class
a.addtask(m)
# set some moment parameters
m.setkey("outfile", "tester")
m.setkey("moments", [0, 1, 2])
print s.image.images
m.addInput(s)
# output filenames will be:
# tester.integrated mom=0
# tester.weighted_coord mom=1
# weighted_dispersion_coord mom=2
# run admit (specifically the tasks that need it
if False:
m.execute()
else:
a.run()
# save it out to disk (this will not be needed soon as I a working on
# a way to write out the xml inside of the run commmand
a.write()
print "ALL DONE. NOW READING BACK"
a2 = ad.Admit() # read in the admit.xml and bdp.xml files
# this should print out something reasonable ending in .xml
#print a.tasks[0].out[0].xmlFile
#print a2.tasks[0].out[0].xmlFile
print "These pairs should match"
for at in a.fm.tasks:
print "FlowManager tasks ", a.fm.tasks
print "FlowManager tasks ", a2.fm.tasks
print "LEN ", len(a.fm.tasks[at].bdp_out)
print "LEN ", len(a2.fm.tasks[at].bdp_out)
print "Input ", a.fm.tasks[at].bdp_in[0].taskid
print "Input ", a2.fm.tasks[at].bdp_in[0].taskid
print "\n\n"
print "Conn map ", a.fm._connmap
print "Conn map ", a2.fm._connmap
print "\n\n"
print "Conn map ", a.fm._depsmap
print "Conn map ", a2.fm._depsmap
print "\n\n"
for at in a.fm.tasks:
for i in a.fm.tasks[at].bdp_out:
if (i.xmlFile == a2.fm.tasks[at].bdp_out[0].xmlFile):
print "File ", i.xmlFile
print "File ", a2.fm.tasks[at].bdp_out[0].xmlFile
print "\n\nPASS\n"
print "running a2 again:"
a2.run()
return
print "\n\nFAIL\n"
def convert(self,
chan=None,
freq=None,
velocity=None,
spec=None,
file=None,
separator=None,
restfreq=None,
vlsr=None):
""" Method to convert input data (either files or arrays) into a CubeSpectrum_BDP. If files
are used then then the columns containing the frequency and the intensity must be given
(channel numbers are optional). Any number of files can be given, but all spectra must
have the same length as they are assumed to come from the same data source. Blank lines
and lines starting with a comment '#' will be skipped, additionally any line with too
few columns will be skipped. If arrays are used an input then both the frequency and
intensity must be specified (the channel numbers are optional). Both lists and numpy
arrays are accepted as inputs. Multidimmensional arrays are supported with the following
parameters:
+ A single frequency list can be given to cover all input spectra, otherwise the shape
of the frequency array must match that of the spectra
+ A single channel list can be given to cover all input spectra, otherwise the shape
of the channel array must match that of the spectra
+ All spectra must have the same length
If a channel array is not specified then one will be constructed with the following
parameters:
+ The channel numbers will start at 0 (casa convention)
+ The first entry in the spectrum will be considered the first channel, regardless of
whether the frequency array increases or decreases.
Additionally, if there is velocity axis, but no frequency axis, a frequency axis can
be constructed by specifying a rest frequency (restfreq), and vlsr.
The convert method will return a single CubeSpectrum_BDP instance holding all input spectra
along with an image of each.
Parameters
----------
chan : array or int
An array holding the channel numbers for the data, multidimmensional arrays are
supported. If an integer is specified then it is the number of the column
in the file which contains the channel numbers, column numbers are 1 based.
Default: None
freq : array
An array holding the frequencies for the data, multidimmensional arrays are
supported. If an integer is specified then it is the number of the column
in the file which contains the frequencies, column numbers are 1 based.
Default: None
velocity : array
An array holding the velocity for the data, multidimmensional arrays are
supported. If an integer is specified then it is the number of the column
in the file which contains the velcoties, column numbers are 1 based. If this
parameter is specified then restfreq and vlsr must also be specified.
Default: None
spec : array
An array holding the intesities of the data, multidimmensional arrays are supported.
If an integer is specified then it is the number of the column in the file which
contains the intensities, column numbers are 1 based.
Default: None
file : list or str
A single file name or a list of file names to be read in for spectra.
Default: None
separator : str
The column separator for reading in the data files.
Default: None (any whitespace)
restfreq : float
The rest frequency to use to convert the spectra from velocity to frequency units.
The rest frequency is in GHz.
Default: None (no conversion done)
vlsr : float
The reference velocity for converting a velocity axis to frequency. The units are
km/s. If this is not set then it is assumed that the vlsr is 0.0.
Default: None
Returns
-------
CubeSpectrum_BDP instance containing all of the inpur spectra.
"""
self.restfreq = restfreq
self.vlsr = vlsr
# if a string was given as the file name then turn it into a list so it can be iterated over
if isinstance(file, str):
self.file = [file]
else:
self.file = file
# do some error checking
if isinstance(chan, np.ndarray) or isinstance(chan, list):
if isinstance(chan, list):
self.chan = np.array(chan)
else:
self.chan = copy.deepcopy(chan)
self.chancol = -1
elif isinstance(chan, int):
self.chancol = chan
self.chan = None
else:
self.chancol = -1
self.chan = None
if isinstance(freq, np.ndarray) or isinstance(freq, list):
if isinstance(freq, list):
self.freq = np.array(freq)
else:
self.freq = copy.deepcopy(freq)
self.freqcol = -1
elif isinstance(freq, int):
self.freqcol = freq
self.freq = None
else:
self.freqcol = -1
self.freq = None
if isinstance(velocity, np.ndarray) or isinstance(velocity, list):
if isinstance(velocity, list):
self.freq = np.array(velocity, dtype=np.float)
else:
self.freq = velocity.astype(np.float)
for i, frq in enumerate(self.freq):
self.freq[i] = self.restfreq + utils.veltofreq(
frq - self.vlsr, self.restfreq)
self.freqcol = -1
elif isinstance(velocity, int):
self.velcol = velocity
self.velocity = None
else:
self.velcol = -1
self.velocity = None
if isinstance(spec, np.ndarray) or isinstance(spec, list):
if isinstance(spec, list):
self.spec = np.array(spec)
else:
self.spec = copy.deepcopy(spec)
self.speccol = -1
elif isinstance(spec, int):
self.speccol = spec
self.spec = None
else:
self.speccol = -1
self.spec = None
if isinstance(separator, str):
self.separator = separator
spectra = []
# read in the data from any files
if self.file:
for fl in self.file:
spectra.append(self.getfile(fl))
else:
# convert the input arrays
singlefreq = False
singlechan = False
havechan = False
# make sure they have the same shape or that the frequency array is 1D
if self.spec.shape != self.freq.shape:
if len(self.spec.shape) == 1 and len(self.freq.shape) != 1:
raise Exception(
"Frequency axis and spectral axis do not have the same shape."
)
else:
singlefreq = True
# make sure they have the same shape or that the channel array is 1D
if self.chan:
havechan = True
if self.spec.shape != self.chan.shape:
if len(spec.shape) == 1 and len(self.chan.shape) != 1:
raise Exception(
"Channel axis and spectral axis do not have the same shape."
)
else:
singlechan = True
# if the arrays are more than 1D, then go through each
if len(self.spec.shape) > 1:
for i in range(self.spec.shape[0]):
spec = self.spec[i]
if not havechan:
chan = np.arange(len(spec))
elif singlechan:
chan = self.chan
else:
chan = self.chan[i]
if singlefreq:
freq = self.freq
else:
freq = self.freq[i]
spectra.append(Spectrum(spec=spec, freq=freq, chans=chan))
else:
# construct the channel array if needed
if not havechan:
self.chan = np.arange(len(self.spec))
spectra.append(
Spectrum(spec=self.spec, freq=self.freq, chans=self.chan))
first = True
images = {}
# make images from the spectra
for i, spec in enumerate(spectra):
data = (spec.chans(masked=False), spec.freq(masked=False),
spec.spec(csub=False, masked=False))
if first:
table = Table(columns=["channel", "frequency", "flux"],
units=["number", "GHz", "Unknown"],
data=np.column_stack(data),
planes=["0"])
first = False
else:
table.addPlane(np.column_stack(data), "%i" % i)
myplot = APlot(ptype=admit.PlotControl.PNG,
pmode=admit.PlotControl.BATCH,
abspath=os.getcwd())
myplot.plotter(spec.freq(masked=False),
[spec.spec(csub=False, masked=False)],
title="Spectrum %i" % i,
figname="fig_%i" % i,
xlab="Frequency",
ylab="Intensity",
thumbnail=True)
# Why not use p1 as the key?
images["fig%i" % i] = myplot.getFigure(figno=myplot.figno,
relative=True)
image = Image(images=images, description="Spectra")
# construct the BDP
bdp = CubeSpectrum_BDP(image=image, table=table)
return bdp
def run(self):
""" The run method, creates the slices, regrids if requested, and
creates the BDP(s)
Parameters
----------
None
Returns
-------
None
"""
dt = utils.Dtime("LineCube")
self._summary = {}
# look for an input noise level, either through keyword or input
# CubeStats BDP or calculate it if needed
pad = self.getkey("pad")
fpad = self.getkey("fpad")
equalize = self.getkey("equalize")
minchan = 0
linelist = self._bdp_in[1]
if linelist == None or len(linelist) == 0:
logging.info("No lines found in input LineList_BDP, exiting.")
return
spw = self._bdp_in[0]
# get the columns from the table
cols = linelist.table.getHeader()
# get the casa image
imagename = spw.getimagefile(bt.CASA)
imh = imhead(self.dir(imagename), mode='list')
# set the overall parameters for imsubimage
args = {"imagename": self.dir(imagename), "overwrite": True}
dt.tag("start")
if pad != 0 or fpad > 0:
nchan = imh['shape'][2]
dt.tag("pad")
# if equal size cubes are requested, this will honor the requested pad
if equalize:
start = linelist.table.getColumnByName("startchan")
end = linelist.table.getColumnByName("endchan")
# look for the widest line
for i in range(len(start)):
diff = end[i] - start[i] + 1
if fpad > 0:
minchan = max(minchan, diff * int(1 + 2 * fpad))
else:
minchan = max(minchan, diff + (2 * pad))
dt.tag("equalize")
# get all of the rows in the table
rows = linelist.getall()
delrow = set()
procblend = [0]
# search through looking for blended lines, leave only the strongest from each blend
# in the list
for i, row in enumerate(rows):
if row.blend in procblend:
continue
strongest = -100.
index = -1
indexes = []
blend = row.blend
for j in range(i, len(rows)):
if rows[j].blend != blend:
continue
indexes.append(j)
if rows[j].linestrength > strongest:
strongest = rows[j].linestrength
index = j
indexes.remove(index)
delrow = delrow | set(indexes)
procblend.append(blend)
dr = list(delrow)
dr.sort()
dr.reverse()
for row in dr:
del rows[row]
# check on duplicate UID's, since those are the directory names here
uid1 = []
for row in rows:
uid1.append(row.getkey("uid"))
uid2 = set(uid1)
if len(uid1) != len(uid2):
print "LineList:", uid1
logging.warning("There are duplicate names in the LineList")
#raise Exception,"There are duplicate names in the LineList"
# Create Summary table
lc_description = admit.util.Table()
lc_description.columns = [
"Line Name", "Start Channel", "End Channel", "Output Cube"
]
lc_description.units = ["", "int", "int", ""]
lc_description.description = "Parameters of Line Cubes"
# loop over all entries in the line list
rdata = []
for row in rows:
uid = row.getkey("uid")
cdir = self.mkext(imagename, uid)
self.mkdir(cdir)
basefl = uid
lcd = [basefl]
outfl = cdir + os.sep + "lc.im"
args["outfile"] = self.dir(outfl)
start = row.getkey("startchan")
end = row.getkey("endchan")
diff = end - start + 1
startch = 0
if diff < minchan:
add = int(math.ceil(float(minchan - diff) / 2.0))
start -= add
end += add
startch += add
if start < 0:
logging.info(
"%s is too close to the edge to encompass with the " +
"requested channels, start=%d resetting to 0" %
(uid, start))
startch += abs(start)
start = 0
if end >= nchan:
logging.info(
"%s is too close to the edge to encompass with the " +
"requested channels, end=%d resetting to %d" %
(uid, end, nchan - 1))
end = nchan - 1
#print "\n\nDIFF ",startch,"\n\n"
if not equalize:
if fpad > 0:
diff = end - start + 1
start -= int(fpad * diff)
end += int(fpad * diff)
if start < 0:
logging.warning(
"fpad=%d too large, start=%d resetting to 0" %
(int(fpad * diff), start))
startch += abs(start)
start = 0
else:
startch += int(fpad * diff)
if end >= nchan:
logging.warning(
"fpad=%d too large, end=%d resetting to %d" %
(int(fpad * diff), end, nchan - 1))
end = nchan - 1
elif pad > 0:
start -= pad
end += pad
if start < 0:
logging.warning(
"pad=%d too large, start=%d resetting to 0" %
(pad, start))
startch += abs(start)
start = 0
else:
startch += pad
if end >= nchan:
logging.warning(
"pad=%d too large, end=%d resetting to %d" %
(pad, end, nchan - 1))
end = nchan - 1
elif pad < 0:
mid = (start + end) / 2
start = mid + pad / 2
end = mid - pad / 2 - 1
if start < 0:
logging.warning(
"pad=%d too large, start=%d resetting to 0" %
(pad, start))
startch += abs(start)
start = 0
else:
startch += abs(start)
if end >= nchan:
logging.warning(
"pad=%d too large, end=%d resetting to %d" %
(pad, end, nchan - 1))
end = nchan - 1
endch = startch + diff
args["chans"] = "%i~%i" % (start, end)
rdata.append(start)
rdata.append(end)
# for the summmary, which will be a table of
# Line name, start channel, end channel, output image
lc_description.addRow([basefl, start, end, outfl])
# create the slices
imsubimage(**args)
line = row.converttoline()
# set the restfrequency ouf the output cube
imhead(imagename=args["outfile"],
mode="put",
hdkey="restfreq",
hdvalue="%fGHz" % (row.getkey("frequency")))
# set up the output BDP
images = {bt.CASA: outfl}
casaimage = Image(images=images)
# note that Summary.getLineFluxes() implicitly relies on the BDP out order
# being the same order as in the line list table. If this is ever not
# true, then Summary.getLineFluxes mismatch BDPs and flux values.
#self.addoutput(LineCube_BDP(xmlFile=cdir + os.sep + basefl + ".lc",
self.addoutput(
LineCube_BDP(xmlFile=outfl,
image=casaimage,
line=line,
linechans="%i~%i" % (startch, endch)))
dt.tag("trans-%s" % cdir)
logging.regression("LC: %s" % str(rdata))
taskargs = "pad=%s fpad=%g equalize=%s" % (pad, fpad, equalize)
self._summary["linecube"] = SummaryEntry(lc_description.serialize(),
"LineCube_AT", self.id(True),
taskargs)
dt.tag("done")
dt.end()
def run(self):
""" The run method, locates lines, attempts to identify them, and
creates the BDP
Parameters
----------
None
Returns
-------
None
"""
if not self.boxcar:
logging.info("Boxcar smoothing turned off.")
self._summary = {}
self.freq = []
self.chan = []
dt = utils.Dtime("LineSegment") # timer for debugging
spec_description = []
taskargs = self._taskargs()
statbdp = None # for the CubeStats BDP
specbdp = None # for the CubeSpectrum BDP
specs = [] # to hold the input CubeSpectrum based spectra
statspec = [] # to hold the input CubeStats based spectrum
statseg = [] # to hold the detected segments from statspec
specseg = [] # to hold the detected segments from specs
#statcutoff = [] # cutoff for statspec line finding
#speccutoff = [] # cutoff for specs line finding
infile = ""
if self.getkey("minchan") < 1:
raise Exception("minchan must eb a positive value.")
elif self.getkey("minchan") == 1 and self.getkey("iterate"):
logging.info(
"iterate=True is not allowed for minchan=1, setting iterate to False"
)
self.setkey("iterate", False)
vlsr = 0.0
# get the input bdp
if self._bdp_in[0] is not None:
specbdp = self._bdp_in[0]
infile = specbdp.xmlFile
if self._bdp_in[1] is not None:
statbdp = self._bdp_in[1]
infile = statbdp.xmlFile
# still need to do this check since all are optional inputs
if specbdp == statbdp is None:
raise Exception("No input BDP's found.")
imbase = self.mkext(infile, 'lseg')
# grab any optional references overplotted on the "ll" plots
# instantiate a plotter for all plots made herein
self._plot_type = admit.util.PlotControl.SVG
myplot = APlot(ptype=self._plot_type,
pmode=self._plot_mode,
abspath=self.dir())
dt.tag("start")
############################################################################
# Smoothing and continuum (baseline) subtraction of input spectra #
############################################################################
# get and smooth all input spectra
basicsegment = {
"method": self.getkey("segment"),
"minchan": self.getkey("minchan"),
"maxgap": self.getkey("maxgap"),
"numsigma": self.getkey("numsigma"),
"iterate": self.getkey("iterate"),
"nomean": True
}
segargsforcont = {
"name": "Line_Segment.%i.asap" % self.id(True),
"pmin": self.getkey("numsigma"),
"minchan": self.getkey("minchan"),
"maxgap": self.getkey("maxgap")
}
if specbdp is not None:
# get the spectrum
specs = specutil.getspectrum(specbdp, vlsr, self.getkey("smooth"),
self.getkey("recalcnoise"),
basicsegment)
# remove the continuum, if requested
if self.getkey("csub")[1] is not None:
logging.info(
"Attempting Continuum Subtraction for Input Spectra")
order = self.getkey("csub")[1]
specutil.contsub(self.id(True),
specs,
self.getkey("segment"),
segargsforcont,
algorithm="PolyFit",
**{"deg": order})
else:
for spec in specs:
spec.set_contin(np.zeros(len(spec)))
for spec in specs:
self.freq, self.chan = specutil.mergefreq(
self.freq, self.chan, spec.freq(False), spec.chans(False))
# get any input cubestats
if statbdp is not None:
statspec = specutil.getspectrum(statbdp, vlsr,
self.getkey("smooth"),
self.getkey("recalcnoise"),
basicsegment)
# remove the continuum
if self.getkey("csub")[0] is not None:
logging.info(
"Attempting Continuum Subtraction for Input CubeStats Spectra"
)
order = self.getkey("csub")[0]
specutil.contsub(self.id(True),
statspec,
self.getkey("segment"),
segargsforcont,
algorithm="PolyFit",
**{"deg": order})
# The 'min' spectrum is inverted for segment finding.
# Doesn't this mean it will also be plotted upside down?
if len(statspec) > 0: statspec[1].invert()
for spec in statspec:
self.freq, self.chan = specutil.mergefreq(
self.freq, self.chan, spec.freq(False), spec.chans(False))
dt.tag("getspectrum")
if isinstance(self.freq, np.ndarray):
self.freq = self.freq.tolist()
if isinstance(self.chan, np.ndarray):
self.chan = self.chan.tolist()
# search for segments of spectral line emission
#NB: this is repetitive with basicsegment above.
method = self.getkey("segment")
minchan = self.getkey("minchan")
maxgap = self.getkey("maxgap")
numsigma = self.getkey("numsigma")
iterate = self.getkey("iterate")
if specbdp is not None:
logging.info("Detecting segments in CubeSpectrum based data")
values = specutil.findsegments(specs, method, minchan, maxgap,
numsigma, iterate)
for i, t in enumerate(values):
specseg.append(t[0])
specs[i].set_noise(t[2])
if statbdp is not None:
logging.info("Detecting segments in CubeStats based data")
values = specutil.findsegments(statspec, method, minchan, maxgap,
numsigma, iterate)
for i, t in enumerate(values):
statseg.append(t[0])
# print ("MWP LINESEGMENT %d Setting noise=%f minchan=%d",(i,t[2],minchan))
statspec[i].set_noise(t[2])
#statcutoff.append(t[1])
dt.tag("segment finder")
lsbdp = LineSegment_BDP(imbase)
finalsegs = utils.mergesegments([statseg, specseg], len(self.freq))
lines = specutil.linedatafromsegments(self.freq, self.chan, finalsegs,
specs, statspec)
llist = []
for l in lines:
lsbdp.addRow(l)
llist.append(l)
rdata = []
# create the output
label = ["Peak/Noise", "Minimum/Noise"]
caption = [
"Potential lines overlaid on peak intensity plot from CubeStats_BDP.",
"Potential lines overlaid on minimum intensity plot from CubeStats_BDP."
]
xlabel = "Frequency (GHz)"
for i, spec in enumerate(statspec):
freqs = []
for ch in statseg[i]:
frq = [
min(spec.freq()[ch[0]],
spec.freq()[ch[1]]),
max(spec.freq()[ch[0]],
spec.freq()[ch[1]])
]
freqs.append(frq)
rdata.append(frq)
#print("Stats segment, peak, ratio, fwhm ",lname,peak,ratio,fwhm)
mult = 1.
if i == 1:
mult = -1.
# print("MWP statspec plot cutoff[%d] = %f, contin=%f" % (i, (statspec[i].contin() + mult*(statspec[i].noise() * self.getkey("numsigma")))[0], statspec[i].contin()[0] ) )
myplot.segplotter(
spec.freq(),
spec.spec(csub=False),
title="Detected Line Segments",
xlab=xlabel,
ylab=label[i],
figname=imbase + "_statspec%i" % i,
segments=freqs,
cutoff=(spec.contin() + mult *
(spec.noise() * self.getkey("numsigma"))),
continuum=spec.contin(),
thumbnail=True)
imname = myplot.getFigure(figno=myplot.figno, relative=True)
thumbnailname = myplot.getThumbnail(figno=myplot.figno,
relative=True)
image = Image(images={bt.SVG: imname},
thumbnail=thumbnailname,
thumbnailtype=bt.PNG,
description=caption[i])
lsbdp.image.addimage(image, "statspec%i" % i)
spec_description.append([
lsbdp.ra, lsbdp.dec, "", xlabel, imname, thumbnailname,
caption[i], infile
])
for i in range(len(specs)):
freqs = []
for ch in specseg[i]:
frq = [
min(specs[i].freq()[ch[0]], specs[i].freq()[ch[1]]),
max(specs[i].freq()[ch[0]], specs[i].freq()[ch[1]])
]
freqs.append(frq)
rdata.append(frq)
myplot.segplotter(specs[i].freq(),
specs[i].spec(csub=False),
title="Detected Line Segments",
xlab=xlabel,
ylab="Intensity",
figname=imbase + "_spec%03d" % i,
segments=freqs,
cutoff=specs[i].contin() +
(specs[i].noise() * self.getkey("numsigma")),
continuum=specs[i].contin(),
thumbnail=True)
imname = myplot.getFigure(figno=myplot.figno, relative=True)
thumbnailname = myplot.getThumbnail(figno=myplot.figno,
relative=True)
caption = "Detected line segments from input spectrum #%i." % (i)
image = Image(images={bt.SVG: imname},
thumbnail=thumbnailname,
thumbnailtype=bt.PNG,
description=caption)
lsbdp.image.addimage(image, "spec%03d" % i)
spec_description.append([
lsbdp.ra, lsbdp.dec, "", xlabel, imname, thumbnailname,
caption, infile
])
caption = "Merged segments overlaid on CubeStats spectrum"
myplot.summaryspec(statspec, specs, None, imbase + "_summary", llist)
imname = myplot.getFigure(figno=myplot.figno, relative=True)
thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True)
caption = "Identified segments overlaid on Signal/Noise plot of all spectra."
image = Image(images={bt.SVG: imname},
thumbnail=thumbnailname,
thumbnailtype=bt.PNG,
description=caption)
lsbdp.image.addimage(image, "summary")
spec_description.append([
lsbdp.ra, lsbdp.dec, "", "Signal/Noise", imname, thumbnailname,
caption, infile
])
self._summary["segments"] = SummaryEntry(lsbdp.table.serialize(),
"LineSegment_AT",
self.id(True), taskargs)
self._summary["spectra"] = [
SummaryEntry(spec_description, "LineSegment_AT", self.id(True),
taskargs)
]
self.addoutput(lsbdp)
logging.regression("LINESEG: %s" % str(rdata))
dt.tag("done")
dt.end()
def run(self):
""" Main program for OverlapIntegral
"""
dt = utils.Dtime("OverlapIntegral")
self._summary = {}
chans =self.getkey("chans")
cmap = self.getkey("cmap")
normalize = self.getkey("normalize")
doCross = True
doCross = False
myplot = APlot(pmode=self._plot_mode,ptype=self._plot_type,abspath=self.dir())
dt.tag("start")
n = len(self._bdp_in)
if n==0:
raise Exception,"Need at least 1 input Image_BDP "
logging.debug("Processing %d input maps" % n)
data = range(n) # array in which each element is placeholder for the data
mdata = range(n) # array to hold the max in each array
summarytable = admit.util.Table()
summarytable.columns = ["File name","Spectral Line ID"]
summarytable.description = "Images used in Overlap Integral"
for i in range(n):
bdpfile = self._bdp_in[i].getimagefile(bt.CASA)
if hasattr(self._bdp_in[i],"line"):
line = getattr(self._bdp_in[i],"line")
logging.info("Map %d: %s" % (i,line.uid))
lineid = line.uid
else:
lineid="no line"
data[i] = casautil.getdata(self.dir(bdpfile),chans)
mdata[i] = data[i].max()
logging.info("shape[%d] = %s with %d good data" % (i,data[i].shape,data[i].count()))
if i==0:
shape = data[i].shape
outfile = self.mkext("testOI","oi")
else:
if shape != data[i].shape:
raise Exception,"Shapes not the same, cannot overlap them"
# collect the file names and line identifications for the summary
summarytable.addRow([bdpfile,lineid])
logging.regression("OI: %s" % str(mdata))
if len(shape)>2 and shape[2] > 1:
raise Exception,"Cannot handle 3D cubes yet"
if doCross:
# debug: produce all cross-corr's of the N input maps (expensive!)
crossn(data, myplot)
dt.tag("crossn")
b1 = Image_BDP(outfile)
self.addoutput(b1)
b1.setkey("image", Image(images={bt.CASA:outfile}))
dt.tag("open")
useClone = True
# to create an output dataset, clone the first input, but using the chans=ch0~ch1
# e.g. using imsubimage(infile,outfile=,chans=
if len(chans) > 0:
# ia.regrid() doesn't have the chans=
taskinit.ia.open(self.dir(self._bdp_in[0].getimagefile(bt.CASA)))
taskinit.ia.regrid(outfile=self.dir(outfile))
taskinit.ia.close()
else:
# 2D for now
if not useClone:
logging.info("OVERLAP out=%s" % outfile)
taskinit.ia.fromimage(infile=self.dir(self._bdp_in[0].getimagefile(bt.CASA)),
outfile=self.dir(outfile), overwrite=True)
taskinit.ia.close()
dt.tag("fromimage")
if n==3:
# RGB
logging.info("RGB mode")
out = rgb1(data[0],data[1],data[2], normalize)
else:
# simple sum
out = data[0]
for i in range(1,n):
out = out + data[i]
if useClone:
casautil.putdata_raw(self.dir(outfile),out,clone=self.dir(self._bdp_in[0].getimagefile(bt.CASA)))
else:
taskinit.ia.open(self.dir(outfile))
s1 = taskinit.ia.shape()
s0 = [0,0,0,0]
r1 = taskinit.rg.box(blc=s0,trc=s1)
pixeldata = out.data
pixelmask = ~out.mask
taskinit.ia.putregion(pixels=pixeldata, pixelmask=pixelmask, region=r1)
taskinit.ia.close()
title = "OverlapIntegral"
pdata = np.rot90(out.squeeze())
logging.info("PDATA: %s" % str(pdata.shape))
myplot.map1(pdata,title,"testOI",thumbnail=True,cmap=cmap)
#-----------------------------
# Populate summary information
#-----------------------------
taskargs = "chans=%s cmap=%s" % (chans, cmap)
imname = ""
thumbnailname = ""
# uncomment when ready.
imname = myplot.getFigure(figno=myplot.figno,relative=True)
thumbnailname = myplot.getThumbnail(figno=myplot.figno,relative=True)
#@todo fill in caption with more info - line names, etc.
caption = "Need descriptive caption here"
summaryinfo = [summarytable.serialize(),imname,thumbnailname,caption]
self._summary["overlap"] = SummaryEntry(summaryinfo,
"OverlapIntegral_AT",
self.id(True),taskargs)
#-----------------------------
dt.tag("done")
dt.end()
def run(self):
#
self._summary = {} # prepare to make a summary here
dt = utils.Dtime("Ingest") # timer for debugging
do_cbeam = True # enforce a common beam
#
pb = self.getkey('pb')
do_pb = len(pb) > 0
use_pb = self.getkey("usepb")
#
create_mask = self.getkey('mask') # create a new mask ?
box = self.getkey("box") # corners in Z, XY or XYZ
edge = self.getkey("edge") # number of edge channels to remove
restfreq = self.getkey("restfreq") # < 0 means not activated
# smooth= could become deprecated, and/or include a decimation option to make it useful
# again, Smooth_AT() does this also , at the cost of an extra cube to store
smooth = self.getkey("smooth") #
#
vlsr = self.getkey("vlsr") # see also LineID, where this could be given again
# first place a fits file in the admit project directory (symlink)
# this is a bit involved, depending on if an absolute or relative path was
# give to Ingest_AT(file=)
fitsfile = self.getkey('file')
if fitsfile[0] != os.sep:
fitsfile = os.path.abspath(os.getcwd() + os.sep + fitsfile)
logging.debug('FILE=%s' % fitsfile)
if fitsfile[0] != os.sep:
raise Exception,"Bad file=%s, expected absolute name",fitsfile
# now determine if it could have been a CASA (or MIRIAD) image already
# which we'll assume if it's a directory; this is natively supported by CASA
# but there are tools where if you pass it a FITS or MIRIAD
# MIRIAD is not recommended for serious work, especially big files, since there
# is a performance penalty due to tiling.
file_is_casa = casautil.iscasa(fitsfile)
loc = fitsfile.rfind(os.sep) # find the '/'
ffile0 = fitsfile[loc+1:] # basename.fits
basename = self.getkey('basename') # (new) basename allowed (allow no dots?)
if len(basename) == 0:
basename = ffile0[:ffile0.rfind('.')] # basename
logging.info("basename=%s" % basename)
target = self.dir(ffile0)
if not os.path.exists(target) :
cmd = 'ln -s "%s" "%s"' % (fitsfile, target)
logging.debug("CMD: %s" % cmd)
os.system(cmd)
readonly = False
if file_is_casa:
logging.debug("Assuming input %s is a CASA (or MIRIAD) image" % ffile0)
bdpfile = self.mkext(basename,"im")
if bdpfile == ffile0:
logging.warning("No selections allowed on CASA image, since no alias was given")
readonly = True
b1 = SpwCube_BDP(bdpfile)
self.addoutput(b1)
b1.setkey("image", Image(images={bt.CASA:bdpfile}))
# @todo b2 and PB?
else:
# construct the output name and construct the BDP based on the CASA image name
# this also takes care of the behind the scenes alias= substitution
bdpfile = self.mkext(basename,"im")
if bdpfile == basename:
raise Exception,"basename and bdpfile are the same, Ingest_AT needs a fix for this"
b1 = SpwCube_BDP(bdpfile)
self.addoutput(b1)
if do_pb:
print "doing the PB"
bdpfile2 = self.mkext(basename,"pb")
b2 = Image_BDP(bdpfile2)
self.addoutput(b2)
# @todo we should also set readonly=True if no box, no mask etc. and still an alias
# that way it will speed up and not make a copy of the image ?
# fni and fno are full (abspath) filenames, ready for CASA
# fni is the same as fitsfile
fni = self.dir(ffile0)
fno = self.dir(bdpfile)
if do_pb: fno2 = self.dir(bdpfile2)
dt.tag("start")
if file_is_casa:
taskinit.ia.open(fni)
else:
if do_pb and use_pb:
# @todo this needs a fix for the path for pb, only works if abs path is given
# impbcor(im.fits,pb.fits,out.im,overwrite=True,mode='m')
if False:
# this may seem like a nice shortcut, to have the fits->casa conversion be done
# internally in impbcor, but it's a terrible performance for big cubes. (tiling?)
# we keep this code here, perhaps at some future time (mpi?) this performs better
# @todo fno2
impbcor(fni,pb,fno,overwrite=True,mode='m')
dt.tag("impbcor-1")
else:
# the better way is to convert FITS->CASA and then call impbcor()
# the CPU savings are big, but I/O overhead can still be substantial
taskinit.ia.fromfits('_pbcor',fni,overwrite=True)
taskinit.ia.fromfits('_pb',pb,overwrite=True)
dt.tag("impbcor-1f")
if False:
impbcor('_pbcor','_pb',fno,overwrite=True,mode='m')
# @todo fno2
utils.remove('_pbcor')
utils.remove('_pb')
dt.tag("impbcor-2")
else:
# immath appears to be even faster (2x in CPU)
# https://bugs.nrao.edu/browse/CAS-8299
# @todo this needs to be confirmed that impbcor is now good to go (r36078)
casa.immath(['_pbcor','_pb'],'evalexpr',fno,'IM0*IM1')
dt.tag("immath")
if True:
# use the mean of all channels... faster may be to use the middle plane
# barf; edge channels can be with fewer subfields in a mosaic
taskinit.ia.open('_pb')
taskinit.ia.summary()
ia1=taskinit.ia.moments(moments=[-1],drop=True,outfile=fno2)
ia1.done()
taskinit.ia.close()
dt.tag("moments")
utils.remove('_pbcor')
utils.remove('_pb')
dt.tag("impbcor-3")
elif do_pb and not use_pb:
# cheat case: PB was given, but not meant to be used
# not implemented yet
print "cheat case dummy PB not implemented yet"
else:
# no PB given
if True:
# re-running this was more consistently faster in wall clock time
# note that zeroblanks=True will still keep the mask
logging.debug("casa::ia.fromfits(%s) -> %s" % (fni,bdpfile))
taskinit.ia.fromfits(fno,fni,overwrite=True)
#taskinit.ia.fromfits(fno,fni,overwrite=True,zeroblanks=True)
dt.tag("fromfits")
else:
# not working to extend 3D yet, but this would solve the impv() 3D problem
logging.debug("casa::importfits(%s) -> %s" % (fni,bdpfile))
#casa.importfits(fni,fno,defaultaxes=True,defaultaxesvalues=[None,None,None,'I'])
# possible bug: zeroblanks=True has no effect?
casa.importfits(fni,fno,zeroblanks=True)
dt.tag("importfits")
taskinit.ia.open(fno)
if len(smooth) > 0:
# smooth here, but Smooth_AT is another option
# here we only allow pixel smoothing
# spatial: gauss
# spectral: boxcar/hanning (check for flux conservation)
# is the boxcar wrong, not centered, but edged?
# @todo CASA BUG: this will loose the object name (and maybe more?) from header, so VLSR lookup fails
fnos = fno + '.smooth'
taskinit.ia.convolve2d(outfile=fnos, overwrite=True, pa='0deg',
major='%gpix' % smooth[0], minor='%gpix' % smooth[1], type='gaussian')
taskinit.ia.close()
srcname = casa.imhead(fno,mode="get",hdkey="object") # work around CASA bug
#@todo use safer ia.rename() here.
# https://casa.nrao.edu/docs/CasaRef/image.rename.html
utils.rename(fnos,fno)
casa.imhead(fno,mode="put",hdkey="object",hdvalue=srcname) # work around CASA bug
dt.tag("convolve2d")
if len(smooth) > 2 and smooth[2] > 0:
if smooth[2] == 1:
# @todo only 1 channel option
specsmooth(fno,fnos,axis=2,function='hanning',dmethod="")
else:
# @todo may have the wrong center
specsmooth(fno,fnos,axis=2,function='boxcar',dmethod="",width=smooth[2])
#@todo use safer ia.rename() here.
# https://casa.nrao.edu/docs/CasaRef/image.rename.html
utils.rename(fnos,fno)
dt.tag("specsmooth")
taskinit.ia.open(fno)
s = taskinit.ia.summary()
if len(s['shape']) != 4:
logging.warning("Adding dummy STOKES-I axis")
fnot = fno + '_4'
taskinit.ia.adddegaxes(stokes='I',outfile=fnot)
taskinit.ia.close()
#@todo use safer ia.rename() here.
# https://casa.nrao.edu/docs/CasaRef/image.rename.html
utils.rename(fnot,fno)
taskinit.ia.open(fno)
dt.tag("adddegaxes")
else:
logging.info("SHAPE: %s" % str(s['shape']))
s = taskinit.ia.summary()
dt.tag("summary-0")
if s['hasmask'] and create_mask:
logging.warning("no extra mask created because input image already had one")
create_mask = False
# if a box= or edge= was given, only a subset of the cube needs to be ingested
# this however complicates PB correction later on
if len(box) > 0 or len(edge) > 0:
if readonly:
raise Exception,"Cannot use box= or edge=, data is read-only, or use an basename/alias"
if len(edge) == 1: edge.append(edge[0])
nx = s['shape'][0]
ny = s['shape'][1]
nz = s['shape'][2]
logging.info("box=%s edge=%s processing with SHAPE: %s" % (str(box),str(edge),str(s['shape'])))
if len(box) == 2:
# select zrange
if len(edge)>0:
raise Exception,"Cannot use edge= when box=[z1,z2] is used"
r1 = taskinit.rg.box([0,0,box[0]] , [nx-1,ny-1,box[1]])
elif len(box) == 4:
if len(edge) == 0:
# select just an XY box
r1 = taskinit.rg.box([box[0],box[1]] , [box[2],box[3]])
elif len(edge) == 2:
# select an XY box, but remove some edge channels
r1 = taskinit.rg.box([box[0],box[1],edge[0]] , [box[2],box[3],nz-edge[1]-1])
else:
raise Exception,"Bad edge= for len(box)=4"
elif len(box) == 6:
# select an XYZ box
r1 = taskinit.rg.box([box[0],box[1],box[2]] , [box[3],box[4],box[5]])
elif len(edge) == 2:
# remove some edge channels, but keep the whole XY box
r1 = taskinit.rg.box([0,0,edge[0]] , [nx-1,ny-1,nz-edge[1]-1])
else:
raise Exception,"box=%s illegal" % box
logging.debug("BOX/EDGE selection: %s %s" % (str(r1['blc']),str(r1['trc'])))
#if taskinit.ia.isopen(): taskinit.ia.close()
logging.info("SUBIMAGE")
subimage = taskinit.ia.subimage(region=r1,outfile=fno+'.box',overwrite=True)
taskinit.ia.close()
taskinit.ia.done()
subimage.rename(fno,overwrite=True)
subimage.close()
subimage.done()
taskinit.ia.open(fno)
dt.tag("subimage-1")
else:
# the whole cube is passed onto ADMIT
if readonly and create_mask:
raise Exception,"Cannot use mask=True, data read-only, or use an alias"
if file_is_casa and not readonly:
# @todo a miriad file - which should be read only - will also create a useless copy here if no alias used
taskinit.ia.subimage(overwrite=True,outfile=fno)
taskinit.ia.close()
taskinit.ia.open(fno)
dt.tag("subimage-0")
if create_mask:
if readonly:
raise Exception,"Cannot create mask, data read-only, or use an alias"
# also check out the 'fromfits::zeroblanks = False'
# calcmask() will overwrite any previous pixelmask
#taskinit.ia.calcmask('mask("%s") && "%s" != 0.0' % (fno,fno))
taskinit.ia.calcmask('"%s" != 0.0' % fno)
dt.tag("mask")
s = taskinit.ia.summary()
dt.tag("summary-1")
# do a fast statistics (no median or robust)
s0 = taskinit.ia.statistics()
dt.tag("statistics")
if len(s0['npts']) == 0:
raise Exception,"No statistics possible, are there valid data in this cube?"
# There may be multiple beams per plane so we can't
# rely on the BEAM's 'major', 'minor', 'positionangle' being present.
# ia.commonbeam() is guaranteed to return beam parameters
# if present
if do_cbeam and s.has_key('perplanebeams'):
# report on the beam extremities, need to loop over all,
# first and last don't need to be extremes....
n = s['perplanebeams']['nChannels']
ab0 = '*0'
bb0 = s['perplanebeams']['beams'][ab0]['*0']
bmaj0 = bb0['major']['value']
bmin0 = bb0['minor']['value']
beamd = 0.0
for i in range(n):
ab1 = '*%d' % i
bb1 = s['perplanebeams']['beams'][ab1]['*0']
bmaj1 = bb1['major']['value']
bmin1 = bb1['minor']['value']
beamd = max(beamd,abs(bmaj0-bmaj1),abs(bmin0-bmin1))
logging.warning("MAX-BEAMSPREAD %f" % (beamd))
#
if True:
logging.info("Applying a commonbeam from the median beam accross the band")
# imhead is a bit slow; alternatively use ia.summary() at the half point for setrestoringbeam()
h = casa.imhead(fno,mode='list')
b = h['perplanebeams']['median area beam']
taskinit.ia.setrestoringbeam(remove=True)
taskinit.ia.setrestoringbeam(beam=b)
commonbeam = taskinit.ia.commonbeam()
else:
# @todo : this will be VERY slow - code not finished, needs renaming etc.
# this is however formally the better solution
logging.warning("commmonbeam code not finished")
cb = taskinit.ia.commonbeam()
taskinit.ia.convolve2d(outfile='junk-common.im', major=cb['major'], minor=cb['minor'], pa=cb['pa'],
targetres=True, overwrite=True)
dt.tag('convolve2d')
commonbeam = {}
else:
try:
commonbeam = taskinit.ia.commonbeam()
except:
nppb = 4.0
logging.warning("No synthesized beam found, faking one to prevent downstream problems: nppb=%f" % nppb)
s = taskinit.ia.summary()
cdelt2 = abs(s['incr'][0]) * 180.0/math.pi*3600.0
bmaj = nppb * cdelt2 # use a nominal 4 points per (round) beam
bmin = nppb * cdelt2
bpa = 0.0
taskinit.ia.setrestoringbeam(major='%farcsec' % bmaj, minor='%farcsec' % bmin, pa='%fdeg' % bpa)
commonbeam = {}
logging.info("COMMONBEAM[%d] %s" % (len(commonbeam),str(commonbeam)))
first_point = taskinit.ia.getchunk(blc=[0,0,0,0],trc=[0,0,0,0],dropdeg=True)
logging.debug("DATA0*: %s" % str(first_point))
taskinit.ia.close()
logging.info('BASICS: [shape] npts min max: %s %d %f %f' % (s['shape'],s0['npts'][0],s0['min'][0],s0['max'][0]))
logging.info('S/N (all data): %f' % (s0['max'][0]/s0['rms'][0]))
npix = 1
nx = s['shape'][0]
ny = s['shape'][1]
nz = s['shape'][2]
for n in s['shape']:
npix = npix * n
ngood = int(s0['npts'][0])
fgood = (1.0*ngood)/npix
logging.info('GOOD PIXELS: %d/%d (%f%% good or %f%% bad)' % (ngood,npix,100.0*fgood,100.0*(1 - fgood)))
if s['hasmask']:
logging.warning('MASKS: %s' % (str(s['masks'])))
if not file_is_casa:
b1.setkey("image", Image(images={bt.CASA:bdpfile}))
if do_pb:
b2.setkey("image", Image(images={bt.CASA:bdpfile2}))
# cube sanity: needs to be either 4D or 2D. But p-p-v cube
# alternative: ia.subimage(dropdeg = True)
# see also: https://bugs.nrao.edu/browse/CAS-5406
shape = s['shape']
if len(shape)>3:
if shape[3]>1:
# @todo this happens when you ingest a fits or casa image which is ra-dec-pol-freq
if nz > 1:
msg = 'Ingest_AT: cannot deal with real 4D cubes yet'
logging.critical(msg)
raise Exception,msg
else:
# @todo this is not working yet when the input was a casa image, but ok when fits. go figure.
fnot = fno + ".trans"
if True:
# this works
#@todo use safer ia.rename() here.
# https://casa.nrao.edu/docs/CasaRef/image.rename.html
utils.rename(fno,fnot)
imtrans(fnot,fno,"0132")
utils.remove(fnot)
else:
# this does not work, what the heck
imtrans(fno,fnot,"0132")
#@todo use safer ia.rename() here.
# https://casa.nrao.edu/docs/CasaRef/image.rename.html
utils.rename(fnot,fno)
nz = s['shape'][3]
# get a new summary 's'
taskinit.ia.open(fno)
s = taskinit.ia.summary()
taskinit.ia.close()
logging.warning("Using imtrans, with nz=%d, to fix axis ordering" % nz)
dt.tag("imtrans4")
# @todo ensure first two axes are position, followed by frequency
elif len(shape)==3:
# the current importfits() can do defaultaxes=True,defaultaxesvalues=['', '', '', 'I']
# but then appears to return a ra-dec-pol-freq cube
# this branch probably never happens, since ia.fromfits() will
# properly convert a 3D cube to 4D now !!
# NO: when NAXIS=3 but various AXIS4's are present, that works. But not if it's pure 3D
# @todo box=
logging.warning("patching up a 3D to 4D cube")
raise Exception,"SHOULD NEVER GET HERE"
fnot = fno + ".trans"
casa.importfits(fni,fnot,defaultaxes=True,defaultaxesvalues=['', '', '', 'I'])
utils.remove(fno) # ieck
imtrans(fnot,fno,"0132")
utils.remove(fnot)
dt.tag("imtrans3")
logging.regression('CUBE: %g %g %g %d %d %d %f' % (s0['min'],s0['max'],s0['rms'],nx,ny,nz,100.0*(1 - fgood)))
# if the cube has only 1 plane (e.g. continuum) , create a visual (png or so)
# for 3D cubes, rely on something like CubeSum
if nz == 1:
implot = ImPlot(pmode=self._plot_mode,ptype=self._plot_type,abspath=self.dir())
implot.plotter(rasterfile=bdpfile,figname=bdpfile)
# @todo needs to be registered for the BDP, right now we only have the plot
# ia.summary() doesn't have this easily available, so run the more expensive imhead()
h = casa.imhead(fno,mode='list')
telescope = h['telescope']
# work around CASA's PIPELINE bug/feature? if 'OBJECT' is blank, try 'FIELD'
srcname = h['object']
if srcname == ' ':
logging.warning('FIELD used for OBJECT')
srcname = casa.imhead(fno,mode='get',hdkey='field')
if srcname == False:
# if no FIELD either, we're doomed. yes, this did happen.
srcname = 'Unknown'
casa.imhead(fno,mode="put",hdkey="object",hdvalue=srcname)
h['object'] = srcname
logging.info('TELESCOPE: %s' % telescope)
logging.info('OBJECT: %s' % srcname)
logging.info('REFFREQTYPE: %s' % h['reffreqtype'])
if h['reffreqtype'].find('TOPO')>=0:
msg = 'Ingest_AT: cannot deal with cubes with TOPOCENTRIC frequencies yet - winging it'
logging.warning(msg)
#raise Exception,msg
# Ensure beam parameters are available if there are multiple beams
# If there is just one beam, then we are just overwriting the header
# variables with their identical values.
if len(commonbeam) != 0:
h['beammajor'] = commonbeam['major']
h['beamminor'] = commonbeam['minor']
h['beampa'] = commonbeam['pa']
# cheat add some things that need to be passed to summary....
h['badpixel'] = 1.0-fgood
if vlsr < -999998.0:
vlsr = admit.VLSR().vlsr(h['object'].upper())
h['vlsr'] = vlsr
logging.info("VLSR = %f (from source catalog)" % vlsr)
taskargs = "file=" + fitsfile
if create_mask == True:
taskargs = taskargs + " mask=True"
if len(box) > 0:
taskargs = taskargs + " " + str(box)
if len(edge) > 0:
taskargs = taskargs + " " + str(edge)
r2d = 57.29577951308232
logging.info("RA Axis 1: %f %f %f" % (h['crval1']*r2d,h['cdelt1']*r2d*3600.0,h['crpix1']))
logging.info("DEC Axis 2: %f %f %f" % (h['crval2']*r2d,h['cdelt2']*r2d*3600.0,h['crpix2']))
if nz > 1:
# @todo check if this is really a freq axis (for ALMA it is, but...)
t3 = h['ctype3']
df = h['cdelt3']
fc = h['crval3'] + (0.5*(float(shape[2])-1)-h['crpix3'])*df # center freq; 0 based pixels
if h.has_key('restfreq'):
fr = float(h['restfreq'][0])
else:
fr = fc
fw = df*float(shape[2])
dv = -df/fr*utils.c
logging.info("Freq Axis 3: %g %g %g" % (h['crval3']/1e9,h['cdelt3']/1e9,h['crpix3']))
logging.info("Cube Axis 3: type=%s velocity increment=%f km/s @ fc=%f fw=%f GHz" % (t3,dv,fc/1e9,fw/1e9))
# @todo sort out this restfreq/vlsr
# report 'reffreqtype', 'restfreq' 'telescope'
# if the fits file has ALTRVAL/ALTRPIX, this is lost in CASA?
# but if you do fits->casa->fits , it's back in fits (with some obvious single precision loss of digits)
# @todo ZSOURCE is the proposed VLSR slot in the fits header, but this has frame issues (it's also optical)
#
# Another method to get the vlsr is to override the restfreq (f0) with an AT keyword
# and the 'restfreq' from the header (f) is then used to compute the vlsr: v = c (1 - f/f0)
#
if shape[2] > 1 and h.has_key('restfreq'):
logging.info("RESTFREQ: %g %g %g" % (fr/1e9,h['restfreq'][0]/1e9,restfreq))
if shape[2] > 1:
# v_radio of the center of the window w.r.t. restfreq
c = utils.c # 299792.458 km/s
vlsrc = c*(1-fc/fr) # @todo rel frame?
vlsrw = dv*float(shape[2])
if restfreq > 0:
vlsrf = c*(1-fr/restfreq/1e9)
h['vlsr'] = vlsrf
else:
vlsrf = 0.0
logging.info("VLSRc = %f VLSRw = %f VLSRf = %f VLSR = %f" % (vlsrc, vlsrw, vlsrf, vlsr))
if h['vlsr'] == 0.0: # @todo! This fails if vlsr actually is zero. Need another magic number
h['vlsr'] = vlsrc
logging.warning("Warning: No VLSR found, substituting VLSRc = %f" % vlsrc)
else:
msg = 'Ingest_AT: missing RESTFREQ'
print msg
# @todo LINTRN is the ALMA keyword that designates the expected line transition in a spw
self._summarize(fitsfile, bdpfile, h, shape, taskargs)
dt.tag("done")
dt.end()