def __init__(self, xmlFile=None, **keyval):
Table_BDP.__init__(self, xmlFile)
Image_BDP.__init__(self, xmlFile)
self.veltype = "vlsr"
self.ra = ""
self.dec = ""
self.table.setkey("columns", utils.linelist_columns)
self.table.setkey("units", utils.linelist_units)
self.table.description = "Identified Spectral Lines"
self.table.data = np.array([], dtype=object)
self.spectra = Table()
self.spectra.setkey("columns", [
"channel", "frequency", "intensity", "mask", "continuum", "noise"
])
self.spectra.setkey("units", ["", "GHz", "", "", "", ""])
self.setkey(keyval)
self._version = "0.2.0"
def __init__(self, xmlFile=None, **keyval):
Table_BDP.__init__(self, xmlFile)
Image_BDP.__init__(self, xmlFile)
self.veltype = "vlsr"
self.ra = ""
self.dec = ""
self.table.setkey("columns", utils.linelist_columns)
self.table.setkey("units", utils.linelist_units)
self.table.description="Identified Spectral Lines"
self.table.data = np.array([], dtype=object)
self.spectra = Table()
self.spectra.setkey("columns", ["channel", "frequency", "intensity",
"mask", "continuum", "noise"])
self.spectra.setkey("units", ["", "GHz", "", "", "", ""])
self.setkey(keyval)
self._version= "0.2.0"
def run(self):
dt = utils.Dtime("PVCorr")
self._summary = {}
numsigma = self.getkey("numsigma")
mode = 1 # PV corr mode (1,2,3)
normalize = True
# normalize = False
b1 = self._bdp_in[0] # PVSlice_BDP
fin = b1.getimagefile(bt.CASA) # CASA image
data = casautil.getdata_raw(
self.dir(fin)) # grab the data as a numpy array
self.myplot = APlot(ptype=self._plot_type,
pmode=self._plot_mode,
abspath=self.dir())
#print 'DATA[0,0]:',data[0,0]
#print 'pv shape: ',data.shape
npos = data.shape[0]
nvel = data.shape[1]
dt.tag("getdata")
b2 = self._bdp_in[1] # CubeStats_BDP
sigma = b2.sigma # global sigma in the cube
cutoff = numsigma * sigma
freq = b2.table.getColumnByName("frequency")
chans = self.getkey("range") # range of channels, if used
if len(chans) > 0:
if len(chans) != 2:
logging.fatal("range=%s" % chans)
raise Exception, "range= needs two values, left and right (inclusive) channel"
ch0 = chans[0]
ch1 = chans[1]
else:
nchan = self.getkey("nchan")
imstat0 = casa.imstat(self.dir(fin)) # @todo can use data[] now
xmaxpos = imstat0['maxpos'][0]
ymaxpos = imstat0['maxpos'][1]
logging.info("MAXPOS-VEL %s %g" %
(str(imstat0['maxpos']), imstat0['max'][0]))
if nchan > 0:
# expand around it, later ch0,ch1 will be checked for running off the edge
ch0 = ymaxpos - nchan / 2
ch1 = ymaxpos + nchan / 2
else:
# watershed down to find ch0 and ch1 ?
# this doesn't work well in crowded areas
ch0 = ymaxpos
ch1 = ymaxpos
spmax = data.max(axis=0)
k = spmax.argmax()
n = len(spmax)
logging.debug('spmax %s %d %g' %
(str(spmax.shape), k, spmax[k]))
# find lower cutoff
for i in range(n):
ch0 = ymaxpos - i
if ch0 < 0: break
if spmax[ch0] < cutoff: break
ch0 = ch0 + 1
# find higher cutoff
for i in range(n):
ch1 = ymaxpos + i
if ch1 == n: break
if spmax[ch1] < cutoff: break
ch1 = ch1 - 1
dt.tag("imstat")
bdp_name = self.mkext(fin, "pvc") # output PVCorr_BDP
b3 = PVCorr_BDP(bdp_name)
self.addoutput(b3)
if ch0 < 0 or ch1 >= nvel:
# this probably only happens to small cubes (problematic for PVCorr)
# or when the strongest line is really close to the edge of the band
# (which is probably ok)
if ch0 < 0 and ch1 >= nvel:
logging.warning("Serious issues with the size of this cube")
if ch0 < 0:
logging.warning("Resetting ch0 edge to 0")
ch0 = 0
if ch1 >= nvel:
ch1 = nvel - 1
logging.warning("Resetting ch1 edge to the maximum")
if ch0 > ch1:
logging.warning("Sanity swapping ch0,1 due to likely noisy data")
ch0, ch1 = ch1, ch0
if mode == 1:
out, rms = mode1(data, ch0, ch1, cutoff, normalize)
corr = out
elif mode == 2:
out, rms = mode2(data, ch0, ch1, cutoff) # slower 2D version
corr = out[
npos /
2, :] # center cut, but could also try feature detection
elif mode == 3:
out, rms = self.mode3(data, ch0, ch1,
cutoff) # Doug's faster 2D version
# get the peak of each column
corr = np.amax(out, axis=0)
# print "PVCORR SHAPE ",corr.shape," mode", mode
if len(corr) > 0:
# print "SHAPE out:",out.shape,corr.shape,npos/2
ch = range(len(corr))
if len(corr) != len(freq):
logging.fatal("ch (%d) and freq (%d) do not have same size" %
(len(corr), len(freq)))
raise Exception, "ch and freq do not have same dimension"
dt.tag("mode")
labels = ["channel", "frequency", "pvcorr"]
units = ["number", "GHz", "N/A"]
data = (ch, freq, corr)
table = Table(columns=labels,
units=units,
data=np.column_stack(data))
else:
# still construct a table, but with no rows
labels = ["channel", "frequency", "pvcorr"]
units = ["number", "GHz", "N/A"]
table = Table(columns=labels, units=units)
b3.setkey("table", table)
b3.setkey("sigma", float(rms))
dt.tag("table")
if len(corr) > 0:
table.exportTable(self.dir("testPVCorr.tab"),
cols=['frequency', 'pvcorr'])
test_single(ch, freq, corr)
logging.regression("PVC: %f %f" % (corr.min(), corr.max()))
title = 'PVCorr mode=%d [%d,%d] %g' % (mode, ch0, ch1, cutoff)
x = ch
xlab = 'Channel'
y = [corr]
ylab = 'PV Correlation'
p1 = "%s_%d" % (bdp_name, 0)
segp = []
segp.append([0, len(ch), 0.0, 0.0])
segp.append([0, len(ch), 3.0 * rms, 3.0 * rms])
# @todo: in principle we know with given noise and size of box, what the sigma in pvcorr should be
self.myplot.plotter(x,
y,
title,
figname=p1,
xlab=xlab,
ylab=ylab,
segments=segp,
thumbnail=True)
#out1 = np.rot90 (data.reshape((nvel,npos)) )
if mode > 1:
self.myplot.map1(data=out,
title="testing PVCorr_AT: mode%d" % mode,
figname='testPVCorr',
thumbnail=True)
taskargs = "numsigma=%.1f range=[%d,%d]" % (numsigma, ch0, ch1)
caption = "Position-velocity correlation plot"
thumbname = self.myplot.getThumbnail(figno=self.myplot.figno,
relative=True)
figname = self.myplot.getFigure(figno=self.myplot.figno,
relative=True)
image = Image(images={bt.PNG: figname},
thumbnail=thumbname,
thumbnailtype=bt.PNG,
description=caption)
b3.image.addimage(image, "pvcorr")
self._summary["pvcorr"] = SummaryEntry(
[figname, thumbname, caption, fin], "PVCorr_AT", self.id(True),
taskargs)
else:
self._summary["pvcorr"] = None
logging.warning("No summary")
logging.regression("PVC: -1")
dt.tag("done")
dt.end()
def run(self):
"""Runs the task.
Parameters
----------
None
Returns
-------
None
"""
self._summary = {}
dt = utils.Dtime("CubeStats")
#maxvrms = 2.0 # maximum variation in rms allowed (hardcoded for now)
#maxvrms = -1.0 # turn maximum variation in rms allowed off
maxvrms = self.getkey("maxvrms")
psample = -1
psample = self.getkey("psample")
# BDP's used :
# b1 = input BDP
# b2 = output BDP
b1 = self._bdp_in[0]
fin = b1.getimagefile(bt.CASA)
bdp_name = self.mkext(fin,'cst')
b2 = CubeStats_BDP(bdp_name)
self.addoutput(b2)
# PeakPointPlot
use_ppp = self.getkey("ppp")
# peakstats: not enabled for mortal users yet
# peakstats = (psample=1, numsigma=4, minchan=3, maxgap=2, peakfit=False)
pnumsigma = 4
minchan = 3
maxgap = 2
peakfit = False # True will enable a true gaussian fit
# numsigma: adding all signal > numsigma ; not user enabled; for peaksum.
numsigma = -1.0
numsigma = 3.0
# grab the new robust statistics. If this is used, 'rms' will be the RMS,
# else we will use RMS = 1.4826*MAD (MAD does a decent job on outliers as well)
# and was the only method available before CASA 4.4 when robust was implemented
robust = self.getkey("robust")
rargs = casautil.parse_robust(robust)
nrargs = len(rargs)
if nrargs == 0:
sumrargs = "medabsdevmed" # for the summary, indicate the default robust
else:
sumrargs = str(rargs)
self._summary["rmsmethd"] = SummaryEntry([sumrargs,fin],"CubeStats_AT",self.id(True))
#@todo think about using this instead of putting 'fin' in all the SummaryEntry
#self._summary["casaimage"] = SummaryEntry(fin,"CubeStats_AT",self.id(True))
# extra CASA call to get the freq's in GHz, as these are not in imstat1{}
# @todo what if the coordinates are not in FREQ ?
# Note: CAS-7648 bug on 3D cubes
if False:
# csys method
ia.open(self.dir(fin))
csys = ia.coordsys()
spec_axis = csys.findaxisbyname("spectral")
# ieck, we need a valid position, or else it will come back and "Exception: All selected pixels are masked"
#freqs = ia.getprofile(spec_axis, region=rg.box([0,0],[0,0]))['coords']/1e9
#freqs = ia.getprofile(spec_axis)['coords']/1e9
freqs = ia.getprofile(spec_axis,unit="GHz")['coords']
dt.tag("getprofile")
else:
# old imval method
#imval0 = casa.imval(self.dir(fin),box='0,0,0,0') # this fails on 3D
imval0 = casa.imval(self.dir(fin))
freqs = imval0['coords'].transpose()[2]/1e9
dt.tag("imval")
nchan = len(freqs)
chans = np.arange(nchan)
# call CASA to get what we want
# imstat0 is the whole cube, imstat1 the plane based statistics
# warning: certain robust stats (**rargs) on the whole cube are going to be very slow
dt.tag("start")
imstat0 = casa.imstat(self.dir(fin), logfile=self.dir('imstat0.logfile'),append=False,**rargs)
dt.tag("imstat0")
imstat1 = casa.imstat(self.dir(fin),axes=[0,1],logfile=self.dir('imstat1.logfile'),append=False,**rargs)
dt.tag("imstat1")
# imm = casa.immoments(self.dir(fin),axis='spec', moments=8, outfile=self.dir('ppp.im'))
if nrargs > 0:
# need to get the peaks without rubust
imstat10 = casa.imstat(self.dir(fin), logfile=self.dir('imstat0.logfile'),append=True)
dt.tag("imstat10")
imstat11 = casa.imstat(self.dir(fin),axes=[0,1],logfile=self.dir('imstat1.logfile'),append=True)
dt.tag("imstat11")
# grab the relevant plane-based things from imstat1
if nrargs == 0:
mean = imstat1["mean"]
sigma = imstat1["medabsdevmed"]*1.4826 # see also: astropy.stats.median_absolute_deviation()
peakval = imstat1["max"]
minval = imstat1["min"]
else:
mean = imstat1["mean"]
sigma = imstat1["rms"]
peakval = imstat11["max"]
minval = imstat11["min"]
if True:
# work around a bug in imstat(axes=[0,1]) for last channel [CAS-7697]
for i in range(len(sigma)):
if sigma[i] == 0.0:
minval[i] = peakval[i] = 0.0
# too many variations in the RMS ?
sigma_pos = sigma[np.where(sigma>0)]
smin = sigma_pos.min()
smax = sigma_pos.max()
logging.info("sigma varies from %f to %f; %d/%d channels ok" % (smin,smax,len(sigma_pos),len(sigma)))
if maxvrms > 0:
if smax/smin > maxvrms:
cliprms = smin * maxvrms
logging.warning("sigma varies too much, going to clip to %g (%g > %g)" % (cliprms, smax/smin, maxvrms))
sigma = np.where(sigma < cliprms, sigma, cliprms)
# @todo (and check again) for foobar.fits all sigma's became 0 when robust was selected
# was this with mask=True/False?
# PeakPointPlot (can be expensive, hence the option)
if use_ppp:
logging.info("Computing MaxPos for PeakPointPlot")
xpos = np.zeros(nchan)
ypos = np.zeros(nchan)
peaksum = np.zeros(nchan)
ia.open(self.dir(fin))
for i in range(nchan):
if sigma[i] > 0.0:
plane = ia.getchunk(blc=[0,0,i,-1],trc=[-1,-1,i,-1],dropdeg=True)
v = ma.masked_invalid(plane)
v_abs = np.absolute(v)
max = np.unravel_index(v_abs.argmax(), v_abs.shape)
xpos[i] = max[0]
ypos[i] = max[1]
if numsigma > 0.0:
peaksum[i] = ma.masked_less(v,numsigma * sigma[i]).sum()
peaksum = np.nan_to_num(peaksum) # put 0's where nan's are found
ia.close()
dt.tag("ppp")
nzeros = len(np.where(sigma<=0.0))
if nzeros > 0:
zeroch = np.where(sigma<=0.0)
logging.warning("There are %d fully masked channels (%s)" % (nzeros,str(zeroch)))
# construct the admit Table for CubeStats_BDP
# note data needs to be a tuple, later to be column_stack'd
if use_ppp:
labels = ["channel" ,"frequency" ,"mean" ,"sigma" ,"max" ,"maxposx" ,"maxposy" ,"min", "peaksum"]
units = ["number" ,"GHz" ,"Jy/beam" ,"Jy/beam" ,"Jy/beam" ,"number" ,"number" ,"Jy/beam", "Jy"]
data = (chans ,freqs ,mean ,sigma ,peakval ,xpos ,ypos ,minval, peaksum)
else:
labels = ["channel" ,"frequency" ,"mean" ,"sigma" ,"max" ,"min"]
units = ["number" ,"GHz" ,"Jy/beam" ,"Jy/beam" ,"Jy/beam" ,"Jy/beam"]
data = (chans ,freqs ,mean ,sigma ,peakval ,minval)
table = Table(columns=labels,units=units,data=np.column_stack(data))
b2.setkey("table",table)
# get the full cube statistics, it depends if robust was pre-selected
if nrargs == 0:
mean0 = imstat0["mean"][0]
sigma0 = imstat0["medabsdevmed"][0]*1.4826
peak0 = imstat0["max"][0]
b2.setkey("mean" , float(mean0))
b2.setkey("sigma", float(sigma0))
b2.setkey("minval",float(imstat0["min"][0]))
b2.setkey("maxval",float(imstat0["max"][0]))
b2.setkey("minpos",imstat0["minpos"][:3].tolist()) #? [] or array(..dtype=int32) ??
b2.setkey("maxpos",imstat0["maxpos"][:3].tolist()) #? [] or array(..dtype=int32) ??
logging.info("CubeMax: %f @ %s" % (imstat0["max"][0],str(imstat0["maxpos"])))
logging.info("CubeMin: %f @ %s" % (imstat0["min"][0],str(imstat0["minpos"])))
logging.info("CubeRMS: %f" % sigma0)
else:
mean0 = imstat0["mean"][0]
sigma0 = imstat0["rms"][0]
peak0 = imstat10["max"][0]
b2.setkey("mean" , float(mean0))
b2.setkey("sigma", float(sigma0))
b2.setkey("minval",float(imstat10["min"][0]))
b2.setkey("maxval",float(imstat10["max"][0]))
b2.setkey("minpos",imstat10["minpos"][:3].tolist()) #? [] or array(..dtype=int32) ??
b2.setkey("maxpos",imstat10["maxpos"][:3].tolist()) #? [] or array(..dtype=int32) ??
logging.info("CubeMax: %f @ %s" % (imstat10["max"][0],str(imstat10["maxpos"])))
logging.info("CubeMin: %f @ %s" % (imstat10["min"][0],str(imstat10["minpos"])))
logging.info("CubeRMS: %f" % sigma0)
b2.setkey("robust",robust)
rms_ratio = imstat0["rms"][0]/sigma0
logging.info("RMS Sanity check %f" % rms_ratio)
if rms_ratio > 1.5:
logging.warning("RMS sanity check = %f. Either bad sidelobes, lotsa signal, or both" % rms_ratio)
logging.regression("CST: %f %f" % (sigma0, rms_ratio))
# plots: no plots need to be made when nchan=1 for continuum
# however we could make a histogram, overlaying the "best" gauss so
# signal deviations are clear?
logging.info('mean,rms,S/N=%f %f %f' % (mean0,sigma0,peak0/sigma0))
if nchan == 1:
# for a continuum/1-channel we only need to stuff some numbers into the _summary
self._summary["chanrms"] = SummaryEntry([float(sigma0), fin], "CubeStats_AT", self.id(True))
self._summary["dynrange"] = SummaryEntry([float(peak0)/float(sigma0), fin], "CubeStats_AT", self.id(True))
self._summary["datamean"] = SummaryEntry([float(mean0), fin], "CubeStats_AT", self.id(True))
else:
y1 = np.log10(ma.masked_invalid(peakval))
y2 = np.log10(ma.masked_invalid(sigma))
y3 = y1-y2
y4 = np.log10(ma.masked_invalid(-minval))
y5 = y1-y4
y = [y1,y2,y3,y4]
title = 'CubeStats: ' + bdp_name+'_0'
xlab = 'Channel'
ylab = 'log(Peak,Noise,Peak/Noise)'
labels = ['log(peak)','log(rms noise)','log(peak/noise)','log(|minval|)']
myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir())
segp = [[chans[0],chans[nchan-1],math.log10(sigma0),math.log10(sigma0)]]
myplot.plotter(chans,y,title,bdp_name+"_0",xlab=xlab,ylab=ylab,segments=segp,labels=labels,thumbnail=True)
imfile = myplot.getFigure(figno=myplot.figno,relative=True)
thumbfile = myplot.getThumbnail(figno=myplot.figno,relative=True)
image0 = Image(images={bt.PNG:imfile},thumbnail=thumbfile,thumbnailtype=bt.PNG,description="CubeStats_0")
b2.addimage(image0,"im0")
if use_ppp:
# new trial for Lee
title = 'PeakSum: (numsigma=%.1f)' % (numsigma)
ylab = 'Jy*N_ppb'
myplot.plotter(chans,[peaksum],title,bdp_name+"_00",xlab=xlab,ylab=ylab,thumbnail=False)
if True:
# hack ascii table
y30 = np.where(sigma > 0, np.log10(peakval/sigma), 0.0)
table2 = Table(columns=["freq","log(P/N)"],data=np.column_stack((freqs,y30)))
table2.exportTable(self.dir("testCubeStats.tab"))
del table2
# the "box" for the "spectrum" is all pixels. Don't know how to
# get this except via shape.
ia.open(self.dir(fin))
s = ia.summary()
ia.close()
if 'shape' in s:
specbox = (0,0,s['shape'][0],s['shape'][1])
else:
specbox = ()
caption = "Emission characteristics as a function of channel, as derived by CubeStats_AT "
caption += "(cyan: global rms,"
caption += " green: noise per channel,"
caption += " blue: peak value per channel,"
caption += " red: peak/noise per channel)."
self._summary["spectra"] = SummaryEntry([0, 0, str(specbox), 'Channel', imfile, thumbfile , caption, fin], "CubeStats_AT", self.id(True))
self._summary["chanrms"] = SummaryEntry([float(sigma0), fin], "CubeStats_AT", self.id(True))
# @todo Will imstat["max"][0] always be equal to s['datamax']? If not, why not?
if 'datamax' in s:
self._summary["dynrange"] = SummaryEntry([float(s['datamax']/sigma0), fin], "CubeStats_AT", self.id(True))
else:
self._summary["dynrange"] = SummaryEntry([float(imstat0["max"][0]/sigma0), fin], "CubeStats_AT", self.id(True))
self._summary["datamean"] = SummaryEntry([imstat0["mean"][0], fin], "CubeStats_AT", self.id(True))
title = bdp_name + "_1"
xlab = 'log(Peak,Noise,P/N)'
myplot.histogram([y1,y2,y3],title,bdp_name+"_1",xlab=xlab,thumbnail=True)
imfile = myplot.getFigure(figno=myplot.figno,relative=True)
thumbfile = myplot.getThumbnail(figno=myplot.figno,relative=True)
image1 = Image(images={bt.PNG:imfile},thumbnail=thumbfile,thumbnailtype=bt.PNG,description="CubeStats_1")
b2.addimage(image1,"im1")
# note that the 'y2' can have been clipped, which can throw off stats.robust()
# @todo should set a mask for those.
title = bdp_name + "_2"
xlab = 'log(Noise))'
n = len(y2)
ry2 = stats.robust(y2)
y2_mean = ry2.mean()
y2_std = ry2.std()
if n>9: logging.debug("NORMALTEST2: %s" % str(scipy.stats.normaltest(ry2)))
myplot.hisplot(y2,title,bdp_name+"_2",xlab=xlab,gauss=[y2_mean,y2_std],thumbnail=True)
title = bdp_name + "_3"
xlab = 'log(diff[Noise])'
n = len(y2)
# dy2 = y2[0:-2] - y2[1:-1]
dy2 = ma.masked_equal(y2[0:-2] - y2[1:-1],0.0).compressed()
rdy2 = stats.robust(dy2)
dy2_mean = rdy2.mean()
dy2_std = rdy2.std()
if n>9: logging.debug("NORMALTEST3: %s" % str(scipy.stats.normaltest(rdy2)))
myplot.hisplot(dy2,title,bdp_name+"_3",xlab=xlab,gauss=[dy2_mean,dy2_std],thumbnail=True)
title = bdp_name + "_4"
xlab = 'log(Signal/Noise))'
n = len(y3)
ry3 = stats.robust(y3)
y3_mean = ry3.mean()
y3_std = ry3.std()
if n>9: logging.debug("NORMALTEST4: %s" % str(scipy.stats.normaltest(ry3)))
myplot.hisplot(y3,title,bdp_name+"_4",xlab=xlab,gauss=[y3_mean,y3_std],thumbnail=True)
title = bdp_name + "_5"
xlab = 'log(diff[Signal/Noise)])'
n = len(y3)
dy3 = y3[0:-2] - y3[1:-1]
rdy3 = stats.robust(dy3)
dy3_mean = rdy3.mean()
dy3_std = rdy3.std()
if n>9: logging.debug("NORMALTEST5: %s" % str(scipy.stats.normaltest(rdy3)))
myplot.hisplot(dy3,title,bdp_name+"_5",xlab=xlab,gauss=[dy3_mean,dy3_std],thumbnail=True)
title = bdp_name + "_6"
xlab = 'log(Peak+Min)'
n = len(y1)
ry5 = stats.robust(y5)
y5_mean = ry5.mean()
y5_std = ry5.std()
if n>9: logging.debug("NORMALTEST6: %s" % str(scipy.stats.normaltest(ry5)))
myplot.hisplot(y5,title,bdp_name+"_6",xlab=xlab,gauss=[y5_mean,y5_std],thumbnail=True)
logging.debug("LogPeak: m,s= %f %f min/max %f %f" % (y1.mean(),y1.std(),y1.min(),y1.max()))
logging.debug("LogNoise: m,s= %f %f %f %f min/max %f %f" % (y2.mean(),y2.std(),y2_mean,y2_std,y2.min(),y2.max()))
logging.debug("LogDeltaNoise: RMS/sqrt(2)= %f %f " % (dy2.std()/math.sqrt(2),dy2_std/math.sqrt(2)))
logging.debug("LogDeltaP/N: RMS/sqrt(2)= %f %f" % (dy3.std()/math.sqrt(2),dy3_std/math.sqrt(2)))
logging.debug("LogPeak+Min: robust m,s= %f %f" % (y5_mean,y5_std))
# compute two ratios that should both be near 1.0 if noise is 'normal'
ratio = y2.std()/(dy2.std()/math.sqrt(2))
ratio2 = y2_std/(dy2_std/math.sqrt(2))
logging.info("RMS BAD VARIATION RATIO: %f %f" % (ratio,ratio2))
# making PPP plot
if nchan > 1 and use_ppp:
smax = 10
gamma = 0.75
z0 = peakval/peakval.max()
# point sizes
s = np.pi * ( smax * (z0**gamma) )**2
cmds = ["grid", "axis equal"]
title = "Peak Points per channel"
pppimage = bdp_name + '_ppp'
myplot.scatter(xpos,ypos,title=title,figname=pppimage,size=s,color=chans,cmds=cmds,thumbnail=True)
pppimage = myplot.getFigure(figno=myplot.figno,relative=True)
pppthumbnail = myplot.getThumbnail(figno=myplot.figno,relative=True)
caption = "Peak point plot: Locations of per-channel peaks in the image cube " + fin
self._summary["peakpnt"] = SummaryEntry([pppimage, pppthumbnail, caption, fin], "CubeStats_AT", self.id(True))
dt.tag("plotting")
# making PeakStats plot
if nchan > 1 and psample > 0:
logging.info("Computing peakstats")
# grab peak,mean and width values for all peaks
(pval,mval,wval) = peakstats(self.dir(fin),freqs,sigma0,pnumsigma,minchan,maxgap,psample,peakfit)
title = "PeakStats: cutoff = %g" % (sigma0*pnumsigma)
xlab = 'Peak value'
ylab = 'FWHM (channels)'
pppimage = bdp_name + '_peakstats'
cval = mval
myplot.scatter(pval,wval,title=title,xlab=xlab,ylab=ylab,color=cval,figname=pppimage,thumbnail=False)
dt.tag("peakstats")
# myplot.final() # pjt debug
# all done!
dt.tag("done")
taskargs = "robust=" + sumrargs
if use_ppp:
taskargs = taskargs + " ppp=True"
else:
taskargs = taskargs + " ppp=False"
for v in self._summary:
self._summary[v].setTaskArgs(taskargs)
dt.tag("summary")
dt.end()
class LineList_BDP(Table_BDP, Image_BDP):
""" LineList BDP class.
This class contains a list of spectral lines identified by the LineID
AT. The columns in the table are: fullname (name of the molecule "U"
for unknown), formula (chemical formula), frequency (rest frequency in
GHz), uid (unique identifier consisting of the formula and rest
frequency), transition (molecular, vibrational or electronic
transition), velocity (relative to the rest velocity), El (lower state
energy in K), Eu (upper state energy in K), linestrength (line
strength of the transition in Debye^2), peakintensity (peak intensity
of the transition in Jy/bm), peakoffset (offset of the peak from rest
in km/s), fwhm (full width half max of the line in km/s), startchan
(starting channel in the spectral window), endchan (ending channel in
the spectral window), and sigma (intensity of the line relative to the
noise level).
Parameters
----------
xmlFile : str
Output XML file name.
keyval : dict
Dictionary of keyword:value pairs.
Attributes
----------
table : Table
Instance of the Table class to hold the spectral line information.
veltype : str
Velocity definition used for the spectrum.
Default: "vlsr"
ra : str
The RA of where the spectrum was taken.
Default: ""
dec : str
The declination of where the spectrum was taken.
Default: ""
spectra : Table
Instance of the Table class to hold spectra.
"""
def __init__(self, xmlFile=None, **keyval):
Table_BDP.__init__(self, xmlFile)
Image_BDP.__init__(self, xmlFile)
self.veltype = "vlsr"
self.ra = ""
self.dec = ""
self.table.setkey("columns", utils.linelist_columns)
self.table.setkey("units", utils.linelist_units)
self.table.description = "Identified Spectral Lines"
self.table.data = np.array([], dtype=object)
self.spectra = Table()
self.spectra.setkey("columns", [
"channel", "frequency", "intensity", "mask", "continuum", "noise"
])
self.spectra.setkey("units", ["", "GHz", "", "", "", ""])
self.setkey(keyval)
self._version = "0.2.0"
def addSpectrum(self, spectrum, name, replace=False):
""" Method to add a spectrum to the BDP
Parameters
----------
spectrum : Spectrum object
The spectrum to add to the BDP
name : str
The name of the spectrum to add (e.g. cubestats)
replace : bool
If True replace the spectrum with the existing name.
Returns
-------
None
"""
# turn the data into a table plane
contin = spectrum.contin(masked=False)
if contin is None:
contin = np.zeros(len(spectrum))
if isinstance(contin, int) or isinstance(contin, float):
contin = np.array([contin] * len(spectrum))
noise = np.array([spectrum.noise()] * len(spectrum))
data = np.column_stack(
(spectrum.chans(False), spectrum.freq(False),
spectrum.spec(csub=False,
masked=False), spectrum.mask(), contin, noise))
# see if a plane already exists with the given name
if name in self.spectra.planes:
if replace:
print "NOT IMPLEMENTED YET"
#self.spectra.replace(name, spectrum)
return
else:
raise Exception("Name %s already exists in Table." % (name))
# if this is the first one
if self.spectra.shape()[0] == 0:
self.spectra.addPlane(data, name)
return
chans = []
chans.append(self.spectra.getColumnByName("channel", typ=np.int32))
# since all planes must have the same shape (numpy restriction) then make sure that they
# all have the same shape before trying to combine them
if len(chans[0]) == len(spectrum.chans()) and chans[0][0] == spectrum.chans(False)[0] and \
chans[0][1] == spectrum.chans()[-1]:
self.spectra.addPlane(data, name)
return
# they are not the same shape (length really) then the shorter ones need to be padded at
# one or both ends
# check for alignment of the channel axis
prependspec = int(max(0, chans[0][0] - spectrum.chans(False)[0]))
appendspec = int(max(0, spectrum.chans(False)[-1] - chans[0][-1]))
prependdata = int(max(0, spectrum.chans(False)[0] - chans[0][0]))
appenddata = int(max(0, chans[0][-1] - spectrum.chans(False)[-1]))
finaldata = {}
# if the plane being added is smaller then pad it by taking the entries from the main table
# setting the spectra to 0.0 and set the mask to True (bad data)
if appenddata != 0 or prependdata != 0:
temp = self.spectra.getPlane(0)
pps = temp[:prependdata]
pps[:, 2] = 0.0
pps[:, 3] = True
pps[:, 4] = 0.0
if appenddata != 0:
aps = temp[-appenddata:]
else:
aps = temp[:0]
aps[:, 2] = 0.0
aps[:, 3] = True
aps[:, 4] = 0.0
finaldata[name] = np.vstack((pps, data, aps))
else:
finaldata[name] = data
# if the main table is smaller then pad all planes by taking the entries from the new plane
# setting the spectra to 0.0 and set the mask to True (bad data)
if prependspec != 0 or appendspec != 0:
spec = {}
if len(self.spectra.shape()) == 2:
spec[self.spectra.planes[0]] = copy.deepcopy(
self.spectra.getPlane(0))
else:
for i in range(self.spectra.shape()[2]):
spec[self.spectra.planes[i]] = copy.deepcopy(
self.spectra.getPlane(i))
self.spectra.clear()
pps = data[:prependspec]
pps[:, 2] = 0.0
pps[:, 3] = True
pps[:, 4] = 0.0
if appendspec != 0:
aps = data[-appendspec:]
else:
aps = data[:0]
aps[:, 2] = 0.0
aps[:, 3] = True
aps[:, 4] = 0.0
for sname, values in spec.iteritems():
finaldata[sname] = np.vstack((pps, values, aps))
# put it all together
for pname, plane in finaldata.iteritems():
self.spectra.addPlane(plane, pname)
def getSpectraNames(self):
""" Method to get the names of the spectra
Parameters
----------
None
Returns
-------
List of strings containing the names
"""
return self.spectra.planes
def getSpectrum(self, name):
""" Method to get a specific spectrum by name
Parameters
----------
name : str
The name of the spectrum to get
Returns
-------
Spectrum instance containing the spectrum
"""
if name not in self.spectra.planes:
raise Exception("Spectrum %s does not exist." % (name))
plane = self.spectra.planes.index(name)
chans = self.spectra.getColumnByName("channel", plane, np.int32)
freq = self.spectra.getColumnByName("frequency", plane, np.float64)
spec = self.spectra.getColumnByName("intensity", plane, np.float64)
mask = self.spectra.getColumnByName("mask", plane, np.bool)
noise = self.spectra.getColumnByName("noise", plane, np.float64)[0]
contin = self.spectra.getColumnByName("continuum", plane, np.float64)
spectrum = Spectrum(spec=spec,
freq=freq,
chans=chans,
mask=mask,
contin=contin,
noise=noise)
return spectrum
def addRow(self, row):
""" Method to add a row to the table
Parameters
----------
row : LineData object
LineData object containing the data
Returns
-------
None
"""
data = []
# build the row from the data
for col in utils.linelist_columns:
data.append(row.getkey(col))
self.table.addRow(data)
def __len__(self):
return len(self.table)
def getall(self):
""" Method to get all rows from the table as a list of LineData objects
Parameters
----------
None
Returns
-------
List of LineData objects, one for each row in the table.
"""
planes = self.getSpectraNames()
tempspec = None
if len(planes) > 0:
tempspec = self.getSpectrum(planes[0])
rows = []
for i in range(len(self)):
row = self.table.getRow(i)
ld = LineData(
name=row[self.table.columns.index("name")],
uid=row[self.table.columns.index("uid")],
transition=row[self.table.columns.index("transition")],
energies=[
row[self.table.columns.index("El")],
row[self.table.columns.index("Eu")]
],
linestrength=float(
row[self.table.columns.index("linestrength")]),
frequency=float(row[self.table.columns.index("frequency")]),
blend=int(row[self.table.columns.index("blend")]),
chans=[
row[self.table.columns.index("startchan")],
row[self.table.columns.index("endchan")]
],
formula=row[self.table.columns.index("formula")],
velocity=row[self.table.columns.index("velocity")],
peakintensity=row[self.table.columns.index("peakintensity")],
peakoffset=row[self.table.columns.index("peakoffset")],
fwhm=row[self.table.columns.index("fwhm")],
peakrms=row[self.table.columns.index("peakrms")],
force=row[self.table.columns.index("force")])
if tempspec is not None:
frqs = [
tempspec.getfreq(
row[self.table.columns.index("startchan")]),
tempspec.getfreq(row[self.table.columns.index("endchan")])
]
frqs.sort()
ld.setkey("freqs", frqs)
rows.append(ld)
return rows
def convert(self, chan=None, freq=None, velocity=None, spec=None, file=None, separator=None,
restfreq=None, vlsr=None):
""" Method to convert input data (either files or arrays) into a CubeSpectrum_BDP. If files
are used then then the columns containing the frequency and the intensity must be given
(channel numbers are optional). Any number of files can be given, but all spectra must
have the same length as they are assumed to come from the same data source. Blank lines
and lines starting with a comment '#' will be skipped, additionally any line with too
few columns will be skipped. If arrays are used an input then both the frequency and
intensity must be specified (the channel numbers are optional). Both lists and numpy
arrays are accepted as inputs. Multidimmensional arrays are supported with the following
parameters:
+ A single frequency list can be given to cover all input spectra, otherwise the shape
of the frequency array must match that of the spectra
+ A single channel list can be given to cover all input spectra, otherwise the shape
of the channel array must match that of the spectra
+ All spectra must have the same length
If a channel array is not specified then one will be constructed with the following
parameters:
+ The channel numbers will start at 0 (casa convention)
+ The first entry in the spectrum will be considered the first channel, regardless of
whether the frequency array increases or decreases.
Additionally, if there is velocity axis, but no frequency axis, a frequency axis can
be constructed by specifying a rest frequency (restfreq), and vlsr.
The convert method will return a single CubeSpectrum_BDP instance holding all input spectra
along with an image of each.
Parameters
----------
chan : array or int
An array holding the channel numbers for the data, multidimmensional arrays are
supported. If an integer is specified then it is the number of the column
in the file which contains the channel numbers, column numbers are 1 based.
Default: None
freq : array
An array holding the frequencies for the data, multidimmensional arrays are
supported. If an integer is specified then it is the number of the column
in the file which contains the frequencies, column numbers are 1 based.
Default: None
velocity : array
An array holding the velocity for the data, multidimmensional arrays are
supported. If an integer is specified then it is the number of the column
in the file which contains the velcoties, column numbers are 1 based. If this
parameter is specified then restfreq and vlsr must also be specified.
Default: None
spec : array
An array holding the intesities of the data, multidimmensional arrays are supported.
If an integer is specified then it is the number of the column in the file which
contains the intensities, column numbers are 1 based.
Default: None
file : list or str
A single file name or a list of file names to be read in for spectra.
Default: None
separator : str
The column separator for reading in the data files.
Default: None (any whitespace)
restfreq : float
The rest frequency to use to convert the spectra from velocity to frequency units.
The rest frequency is in GHz.
Default: None (no conversion done)
vlsr : float
The reference velocity for converting a velocity axis to frequency. The units are
km/s. If this is not set then it is assumed that the vlsr is 0.0.
Default: None
Returns
-------
CubeSpectrum_BDP instance containing all of the inpur spectra.
"""
self.restfreq = restfreq
self.vlsr = vlsr
# if a string was given as the file name then turn it into a list so it can be iterated over
if isinstance(file, str):
self.file = [file]
else:
self.file = file
# do some error checking
if isinstance(chan, np.ndarray) or isinstance(chan, list):
if isinstance(chan, list):
self.chan = np.array(chan)
else:
self.chan = copy.deepcopy(chan)
self.chancol = -1
elif isinstance(chan, int):
self.chancol = chan
self.chan = None
else:
self.chancol = -1
self.chan = None
if isinstance(freq, np.ndarray) or isinstance(freq, list):
if isinstance(freq, list):
self.freq = np.array(freq)
else:
self.freq = copy.deepcopy(freq)
self.freqcol = -1
elif isinstance(freq, int):
self.freqcol = freq
self.freq = None
else:
self.freqcol = -1
self.freq = None
if isinstance(velocity, np.ndarray) or isinstance(velocity, list):
if isinstance(velocity, list):
self.freq = np.array(velocity, dtype=np.float)
else:
self.freq = velocity.astype(np.float)
for i, frq in enumerate(self.freq):
self.freq[i] = self.restfreq + utils.veltofreq(frq - self.vlsr, self.restfreq)
self.freqcol = -1
elif isinstance(velocity, int):
self.velcol = velocity
self.velocity = None
else:
self.velcol = -1
self.velocity = None
if isinstance(spec, np.ndarray) or isinstance(spec, list):
if isinstance(spec, list):
self.spec = np.array(spec)
else:
self.spec = copy.deepcopy(spec)
self.speccol = -1
elif isinstance(spec, int):
self.speccol = spec
self.spec = None
else:
self.speccol = -1
self.spec = None
if isinstance(separator, str):
self.separator = separator
spectra = []
# read in the data from any files
if self.file:
for fl in self.file:
spectra.append(self.getfile(fl))
else:
# convert the input arrays
singlefreq = False
singlechan = False
havechan = False
# make sure they have the same shape or that the frequency array is 1D
if self.spec.shape != self.freq.shape:
if len(self.spec.shape) == 1 and len(self.freq.shape) != 1:
raise Exception("Frequency axis and spectral axis do not have the same shape.")
else:
singlefreq = True
# make sure they have the same shape or that the channel array is 1D
if self.chan:
havechan = True
if self.spec.shape != self.chan.shape:
if len(spec.shape) == 1 and len(self.chan.shape) != 1:
raise Exception("Channel axis and spectral axis do not have the same shape.")
else:
singlechan = True
# if the arrays are more than 1D, then go through each
if len(self.spec.shape) > 1:
for i in range(self.spec.shape[0]):
spec = self.spec[i]
if not havechan:
chan = np.arange(len(spec))
elif singlechan:
chan = self.chan
else:
chan = self.chan[i]
if singlefreq:
freq = self.freq
else:
freq = self.freq[i]
spectra.append(Spectrum(spec=spec, freq=freq, chans=chan))
else:
# construct the channel array if needed
if not havechan:
self.chan = np.arange(len(self.spec))
spectra.append(Spectrum(spec=self.spec, freq=self.freq, chans=self.chan))
first = True
images = {}
# make images from the spectra
for i, spec in enumerate(spectra):
data = (spec.chans(masked=False), spec.freq(masked=False),
spec.spec(csub=False, masked=False))
if first:
table = Table(columns=["channel", "frequency", "flux"],
units=["number", "GHz", "Unknown"], data=np.column_stack(data),
planes=["0"])
first = False
else:
table.addPlane(np.column_stack(data), "%i" % i)
myplot = APlot(ptype=admit.PlotControl.PNG, pmode=admit.PlotControl.BATCH,
abspath=os.getcwd())
myplot.plotter(spec.freq(masked=False), [spec.spec(csub=False, masked=False)],
title="Spectrum %i" % i, figname="fig_%i" % i, xlab="Frequency",
ylab="Intensity", thumbnail=True)
# Why not use p1 as the key?
images["fig%i" % i] = myplot.getFigure(figno=myplot.figno, relative=True)
image = Image(images=images, description="Spectra")
# construct the BDP
bdp = CubeSpectrum_BDP(image=image, table=table)
return bdp
def convert(self,
chan=None,
freq=None,
velocity=None,
spec=None,
file=None,
separator=None,
restfreq=None,
vlsr=None):
""" Method to convert input data (either files or arrays) into a CubeSpectrum_BDP. If files
are used then then the columns containing the frequency and the intensity must be given
(channel numbers are optional). Any number of files can be given, but all spectra must
have the same length as they are assumed to come from the same data source. Blank lines
and lines starting with a comment '#' will be skipped, additionally any line with too
few columns will be skipped. If arrays are used an input then both the frequency and
intensity must be specified (the channel numbers are optional). Both lists and numpy
arrays are accepted as inputs. Multidimmensional arrays are supported with the following
parameters:
+ A single frequency list can be given to cover all input spectra, otherwise the shape
of the frequency array must match that of the spectra
+ A single channel list can be given to cover all input spectra, otherwise the shape
of the channel array must match that of the spectra
+ All spectra must have the same length
If a channel array is not specified then one will be constructed with the following
parameters:
+ The channel numbers will start at 0 (casa convention)
+ The first entry in the spectrum will be considered the first channel, regardless of
whether the frequency array increases or decreases.
Additionally, if there is velocity axis, but no frequency axis, a frequency axis can
be constructed by specifying a rest frequency (restfreq), and vlsr.
The convert method will return a single CubeSpectrum_BDP instance holding all input spectra
along with an image of each.
Parameters
----------
chan : array or int
An array holding the channel numbers for the data, multidimmensional arrays are
supported. If an integer is specified then it is the number of the column
in the file which contains the channel numbers, column numbers are 1 based.
Default: None
freq : array
An array holding the frequencies for the data, multidimmensional arrays are
supported. If an integer is specified then it is the number of the column
in the file which contains the frequencies, column numbers are 1 based.
Default: None
velocity : array
An array holding the velocity for the data, multidimmensional arrays are
supported. If an integer is specified then it is the number of the column
in the file which contains the velcoties, column numbers are 1 based. If this
parameter is specified then restfreq and vlsr must also be specified.
Default: None
spec : array
An array holding the intesities of the data, multidimmensional arrays are supported.
If an integer is specified then it is the number of the column in the file which
contains the intensities, column numbers are 1 based.
Default: None
file : list or str
A single file name or a list of file names to be read in for spectra.
Default: None
separator : str
The column separator for reading in the data files.
Default: None (any whitespace)
restfreq : float
The rest frequency to use to convert the spectra from velocity to frequency units.
The rest frequency is in GHz.
Default: None (no conversion done)
vlsr : float
The reference velocity for converting a velocity axis to frequency. The units are
km/s. If this is not set then it is assumed that the vlsr is 0.0.
Default: None
Returns
-------
CubeSpectrum_BDP instance containing all of the inpur spectra.
"""
self.restfreq = restfreq
self.vlsr = vlsr
# if a string was given as the file name then turn it into a list so it can be iterated over
if isinstance(file, str):
self.file = [file]
else:
self.file = file
# do some error checking
if isinstance(chan, np.ndarray) or isinstance(chan, list):
if isinstance(chan, list):
self.chan = np.array(chan)
else:
self.chan = copy.deepcopy(chan)
self.chancol = -1
elif isinstance(chan, int):
self.chancol = chan
self.chan = None
else:
self.chancol = -1
self.chan = None
if isinstance(freq, np.ndarray) or isinstance(freq, list):
if isinstance(freq, list):
self.freq = np.array(freq)
else:
self.freq = copy.deepcopy(freq)
self.freqcol = -1
elif isinstance(freq, int):
self.freqcol = freq
self.freq = None
else:
self.freqcol = -1
self.freq = None
if isinstance(velocity, np.ndarray) or isinstance(velocity, list):
if isinstance(velocity, list):
self.freq = np.array(velocity, dtype=np.float)
else:
self.freq = velocity.astype(np.float)
for i, frq in enumerate(self.freq):
self.freq[i] = self.restfreq + utils.veltofreq(
frq - self.vlsr, self.restfreq)
self.freqcol = -1
elif isinstance(velocity, int):
self.velcol = velocity
self.velocity = None
else:
self.velcol = -1
self.velocity = None
if isinstance(spec, np.ndarray) or isinstance(spec, list):
if isinstance(spec, list):
self.spec = np.array(spec)
else:
self.spec = copy.deepcopy(spec)
self.speccol = -1
elif isinstance(spec, int):
self.speccol = spec
self.spec = None
else:
self.speccol = -1
self.spec = None
if isinstance(separator, str):
self.separator = separator
spectra = []
# read in the data from any files
if self.file:
for fl in self.file:
spectra.append(self.getfile(fl))
else:
# convert the input arrays
singlefreq = False
singlechan = False
havechan = False
# make sure they have the same shape or that the frequency array is 1D
if self.spec.shape != self.freq.shape:
if len(self.spec.shape) == 1 and len(self.freq.shape) != 1:
raise Exception(
"Frequency axis and spectral axis do not have the same shape."
)
else:
singlefreq = True
# make sure they have the same shape or that the channel array is 1D
if self.chan:
havechan = True
if self.spec.shape != self.chan.shape:
if len(spec.shape) == 1 and len(self.chan.shape) != 1:
raise Exception(
"Channel axis and spectral axis do not have the same shape."
)
else:
singlechan = True
# if the arrays are more than 1D, then go through each
if len(self.spec.shape) > 1:
for i in range(self.spec.shape[0]):
spec = self.spec[i]
if not havechan:
chan = np.arange(len(spec))
elif singlechan:
chan = self.chan
else:
chan = self.chan[i]
if singlefreq:
freq = self.freq
else:
freq = self.freq[i]
spectra.append(Spectrum(spec=spec, freq=freq, chans=chan))
else:
# construct the channel array if needed
if not havechan:
self.chan = np.arange(len(self.spec))
spectra.append(
Spectrum(spec=self.spec, freq=self.freq, chans=self.chan))
first = True
images = {}
# make images from the spectra
for i, spec in enumerate(spectra):
data = (spec.chans(masked=False), spec.freq(masked=False),
spec.spec(csub=False, masked=False))
if first:
table = Table(columns=["channel", "frequency", "flux"],
units=["number", "GHz", "Unknown"],
data=np.column_stack(data),
planes=["0"])
first = False
else:
table.addPlane(np.column_stack(data), "%i" % i)
myplot = APlot(ptype=admit.PlotControl.PNG,
pmode=admit.PlotControl.BATCH,
abspath=os.getcwd())
myplot.plotter(spec.freq(masked=False),
[spec.spec(csub=False, masked=False)],
title="Spectrum %i" % i,
figname="fig_%i" % i,
xlab="Frequency",
ylab="Intensity",
thumbnail=True)
# Why not use p1 as the key?
images["fig%i" % i] = myplot.getFigure(figno=myplot.figno,
relative=True)
image = Image(images=images, description="Spectra")
# construct the BDP
bdp = CubeSpectrum_BDP(image=image, table=table)
return bdp
def run(self):
""" Method to read in a .bdp file and convert it to a BDP object.
Parameters
----------
None
Returns
-------
None
"""
self._summary = {}
if not self.getkey("file"):
raise Exception("Input file name is empty, one must be given.")
bdp = utils.getBDP(self.getkey("file"))
self.addoutput(bdp)
# Make a table of some basic BDP info for Summary.
# Why in god's name do BDPs not store the name of the task that
# created them?!?! The task ID attrbute is useless if the BDP came
# from another flow -- which is why this task was created in
# the first place!
table = Table()
if bdp.project != "":
table.addRow(["Project", bdp.project])
if bdp.sous != "":
table.addRow(["SOUS", bdp.sous])
if bdp.gous != "":
table.addRow(["GOUS", bdp.gous])
if bdp.mous != "":
table.addRow(["MOUS", bdp.mous])
table.addRow(["BDP Type", bdp._type])
table.addRow(["Base directory", bdp._baseDir])
table.addRow(["XML file", bdp.xmlFile])
if bdp._date != "":
table.addRow(["Time stamp", bdp._date])
files = bdp.getfiles()
for f in files:
table.addRow(["Associated File", f])
table.description = "Information about the ingested BDP"
taskargs = "file=%s" % self.getkey('file')
self._summary["bdpingest"] = SummaryEntry(table.serialize(),
"BDPIngest_AT",
self.id(True), taskargs)
class LineList_BDP(Table_BDP, Image_BDP):
""" LineList BDP class.
This class contains a list of spectral lines identified by the LineID
AT. The columns in the table are: fullname (name of the molecule "U"
for unknown), formula (chemical formula), frequency (rest frequency in
GHz), uid (unique identifier consisting of the formula and rest
frequency), transition (molecular, vibrational or electronic
transition), velocity (relative to the rest velocity), El (lower state
energy in K), Eu (upper state energy in K), linestrength (line
strength of the transition in Debye^2), peakintensity (peak intensity
of the transition in Jy/bm), peakoffset (offset of the peak from rest
in km/s), fwhm (full width half max of the line in km/s), startchan
(starting channel in the spectral window), endchan (ending channel in
the spectral window), and sigma (intensity of the line relative to the
noise level).
Parameters
----------
xmlFile : str
Output XML file name.
keyval : dict
Dictionary of keyword:value pairs.
Attributes
----------
table : Table
Instance of the Table class to hold the spectral line information.
veltype : str
Velocity definition used for the spectrum.
Default: "vlsr"
ra : str
The RA of where the spectrum was taken.
Default: ""
dec : str
The declination of where the spectrum was taken.
Default: ""
spectra : Table
Instance of the Table class to hold spectra.
"""
def __init__(self, xmlFile=None, **keyval):
Table_BDP.__init__(self, xmlFile)
Image_BDP.__init__(self, xmlFile)
self.veltype = "vlsr"
self.ra = ""
self.dec = ""
self.table.setkey("columns", utils.linelist_columns)
self.table.setkey("units", utils.linelist_units)
self.table.description="Identified Spectral Lines"
self.table.data = np.array([], dtype=object)
self.spectra = Table()
self.spectra.setkey("columns", ["channel", "frequency", "intensity",
"mask", "continuum", "noise"])
self.spectra.setkey("units", ["", "GHz", "", "", "", ""])
self.setkey(keyval)
self._version= "0.2.0"
def addSpectrum(self, spectrum, name, replace=False):
""" Method to add a spectrum to the BDP
Parameters
----------
spectrum : Spectrum object
The spectrum to add to the BDP
name : str
The name of the spectrum to add (e.g. cubestats)
replace : bool
If True replace the spectrum with the existing name.
Returns
-------
None
"""
# turn the data into a table plane
contin = spectrum.contin(masked=False)
if contin is None:
contin = np.zeros(len(spectrum))
if isinstance(contin, int) or isinstance(contin, float):
contin = np.array([contin] * len(spectrum))
noise = np.array([spectrum.noise()] * len(spectrum))
data = np.column_stack((spectrum.chans(False), spectrum.freq(False),
spectrum.spec(csub=False, masked=False),
spectrum.mask(), contin, noise))
# see if a plane already exists with the given name
if name in self.spectra.planes:
if replace:
print "NOT IMPLEMENTED YET"
#self.spectra.replace(name, spectrum)
return
else:
raise Exception("Name %s already exists in Table." % (name))
# if this is the first one
if self.spectra.shape()[0] == 0:
self.spectra.addPlane(data, name)
return
chans = []
chans.append(self.spectra.getColumnByName("channel", typ=np.int32))
# since all planes must have the same shape (numpy restriction) then make sure that they
# all have the same shape before trying to combine them
if len(chans[0]) == len(spectrum.chans()) and chans[0][0] == spectrum.chans(False)[0] and \
chans[0][1] == spectrum.chans()[-1]:
self.spectra.addPlane(data, name)
return
# they are not the same shape (length really) then the shorter ones need to be padded at
# one or both ends
# check for alignment of the channel axis
prependspec = int(max(0, chans[0][0] - spectrum.chans(False)[0]))
appendspec = int(max(0, spectrum.chans(False)[-1] - chans[0][-1]))
prependdata = int(max(0, spectrum.chans(False)[0] - chans[0][0]))
appenddata = int(max(0, chans[0][-1] - spectrum.chans(False)[-1]))
finaldata = {}
# if the plane being added is smaller then pad it by taking the entries from the main table
# setting the spectra to 0.0 and set the mask to True (bad data)
if appenddata != 0 or prependdata != 0:
temp = self.spectra.getPlane(0)
pps = temp[:prependdata]
pps[:, 2] = 0.0
pps[:, 3] = True
pps[:, 4] = 0.0
if appenddata != 0:
aps = temp[-appenddata:]
else:
aps = temp[:0]
aps[:, 2] = 0.0
aps[:, 3] = True
aps[:, 4] = 0.0
finaldata[name] = np.vstack((pps, data, aps))
else:
finaldata[name] = data
# if the main table is smaller then pad all planes by taking the entries from the new plane
# setting the spectra to 0.0 and set the mask to True (bad data)
if prependspec != 0 or appendspec != 0:
spec = {}
if len(self.spectra.shape()) == 2:
spec[self.spectra.planes[0]] = copy.deepcopy(self.spectra.getPlane(0))
else:
for i in range(self.spectra.shape()[2]):
spec[self.spectra.planes[i]] = copy.deepcopy(self.spectra.getPlane(i))
self.spectra.clear()
pps = data[:prependspec]
pps[:, 2] = 0.0
pps[:, 3] = True
pps[:, 4] = 0.0
if appendspec != 0:
aps = data[-appendspec:]
else:
aps = data[:0]
aps[:, 2] = 0.0
aps[:, 3] = True
aps[:, 4] = 0.0
for sname, values in spec.iteritems():
finaldata[sname] = np.vstack((pps, values, aps))
# put it all together
for pname, plane in finaldata.iteritems():
self.spectra.addPlane(plane, pname)
def getSpectraNames(self):
""" Method to get the names of the spectra
Parameters
----------
None
Returns
-------
List of strings containing the names
"""
return self.spectra.planes
def getSpectrum(self, name):
""" Method to get a specific spectrum by name
Parameters
----------
name : str
The name of the spectrum to get
Returns
-------
Spectrum instance containing the spectrum
"""
if name not in self.spectra.planes:
raise Exception("Spectrum %s does not exist." % (name))
plane = self.spectra.planes.index(name)
chans = self.spectra.getColumnByName("channel", plane, np.int32)
freq = self.spectra.getColumnByName("frequency", plane, np.float64)
spec = self.spectra.getColumnByName("intensity", plane, np.float64)
mask = self.spectra.getColumnByName("mask", plane, np.bool)
noise = self.spectra.getColumnByName("noise", plane, np.float64)[0]
contin = self.spectra.getColumnByName("continuum", plane, np.float64)
spectrum = Spectrum(spec=spec, freq=freq, chans=chans, mask=mask,
contin=contin, noise=noise)
return spectrum
def addRow(self, row):
""" Method to add a row to the table
Parameters
----------
row : LineData object
LineData object containing the data
Returns
-------
None
"""
data = []
# build the row from the data
for col in utils.linelist_columns:
data.append(row.getkey(col))
self.table.addRow(data)
def __len__(self):
return len(self.table)
def getall(self):
""" Method to get all rows from the table as a list of LineData objects
Parameters
----------
None
Returns
-------
List of LineData objects, one for each row in the table.
"""
planes = self.getSpectraNames()
tempspec = None
if len(planes) > 0:
tempspec = self.getSpectrum(planes[0])
rows = []
for i in range(len(self)):
row = self.table.getRow(i)
ld = LineData(name=row[self.table.columns.index("name")],
uid=row[self.table.columns.index("uid")],
transition=row[self.table.columns.index("transition")],
energies=[row[self.table.columns.index("El")], row[self.table.columns.index("Eu")]],
linestrength=float(row[self.table.columns.index("linestrength")]),
frequency=float(row[self.table.columns.index("frequency")]),
blend=int(row[self.table.columns.index("blend")]),
chans=[row[self.table.columns.index("startchan")], row[self.table.columns.index("endchan")]],
formula=row[self.table.columns.index("formula")],
velocity=row[self.table.columns.index("velocity")],
peakintensity=row[self.table.columns.index("peakintensity")],
peakoffset=row[self.table.columns.index("peakoffset")],
fwhm=row[self.table.columns.index("fwhm")],
peakrms=row[self.table.columns.index("peakrms")],
force=row[self.table.columns.index("force")])
if tempspec is not None:
frqs = [tempspec.getfreq(row[self.table.columns.index("startchan")]),
tempspec.getfreq(row[self.table.columns.index("endchan")])]
frqs.sort()
ld.setkey("freqs", frqs)
rows.append(ld)
return rows
def __init__(self, xmlFile=None, **keyval):
BDP.__init__(self, xmlFile)
# instantiate a table as a data member
self.table = Table()
self.setkey(keyval)
self._version = "0.1.0"
def run(self):
dt = utils.Dtime("PVCorr")
self._summary = {}
numsigma = self.getkey("numsigma")
mode = 1 # PV corr mode (1,2,3)
normalize = True
# normalize = False
b1 = self._bdp_in[0] # PVSlice_BDP
fin = b1.getimagefile(bt.CASA) # CASA image
data = casautil.getdata_raw(self.dir(fin)) # grab the data as a numpy array
self.myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir())
#print 'DATA[0,0]:',data[0,0]
#print 'pv shape: ',data.shape
npos = data.shape[0]
nvel = data.shape[1]
dt.tag("getdata")
b2 = self._bdp_in[1] # CubeStats_BDP
sigma = b2.sigma # global sigma in the cube
cutoff = numsigma * sigma
freq = b2.table.getColumnByName("frequency")
chans = self.getkey("range") # range of channels, if used
if len(chans) > 0:
if len(chans) != 2:
logging.fatal("range=%s" % chans)
raise Exception,"range= needs two values, left and right (inclusive) channel"
ch0 = chans[0]
ch1 = chans[1]
else:
nchan = self.getkey("nchan")
imstat0 = casa.imstat(self.dir(fin)) # @todo can use data[] now
xmaxpos = imstat0['maxpos'][0]
ymaxpos = imstat0['maxpos'][1]
logging.info("MAXPOS-VEL %s %g" % (str(imstat0['maxpos']),imstat0['max'][0]))
if nchan > 0:
# expand around it, later ch0,ch1 will be checked for running off the edge
ch0 = ymaxpos - nchan/2
ch1 = ymaxpos + nchan/2
else:
# watershed down to find ch0 and ch1 ?
# this doesn't work well in crowded areas
ch0 = ymaxpos
ch1 = ymaxpos
spmax = data.max(axis=0)
k = spmax.argmax()
n = len(spmax)
logging.debug('spmax %s %d %g' % (str(spmax.shape),k,spmax[k]))
# find lower cutoff
for i in range(n):
ch0 = ymaxpos - i
if ch0<0: break
if spmax[ch0] < cutoff: break
ch0 = ch0 + 1
# find higher cutoff
for i in range(n):
ch1 = ymaxpos + i
if ch1==n: break
if spmax[ch1] < cutoff: break
ch1 = ch1 - 1
dt.tag("imstat")
bdp_name = self.mkext(fin,"pvc") # output PVCorr_BDP
b3 = PVCorr_BDP(bdp_name)
self.addoutput(b3)
if ch0<0 or ch1>=nvel:
# this probably only happens to small cubes (problematic for PVCorr)
# or when the strongest line is really close to the edge of the band
# (which is probably ok)
if ch0<0 and ch1>=nvel:
logging.warning("Serious issues with the size of this cube")
if ch0<0:
logging.warning("Resetting ch0 edge to 0")
ch0=0
if ch1>=nvel:
ch1=nvel-1
logging.warning("Resetting ch1 edge to the maximum")
if ch0 > ch1:
logging.warning("Sanity swapping ch0,1 due to likely noisy data")
ch0,ch1 = ch1,ch0
if mode == 1:
out,rms = mode1(data, ch0, ch1, cutoff, normalize)
corr = out
elif mode == 2:
out,rms = mode2(data, ch0, ch1, cutoff) # slower 2D version
corr = out[npos/2,:] # center cut, but could also try feature detection
elif mode == 3:
out,rms = self.mode3(data, ch0, ch1, cutoff) # Doug's faster 2D version
# get the peak of each column
corr = np.amax(out,axis=0)
# print "PVCORR SHAPE ",corr.shape," mode", mode
if len(corr) > 0:
# print "SHAPE out:",out.shape,corr.shape,npos/2
ch = range(len(corr))
if len(corr) != len(freq):
logging.fatal("ch (%d) and freq (%d) do not have same size" % (len(corr),len(freq)))
raise Exception,"ch and freq do not have same dimension"
dt.tag("mode")
labels = ["channel", "frequency", "pvcorr"]
units = ["number", "GHz", "N/A"]
data = (ch, freq, corr)
table = Table(columns=labels,units=units,data=np.column_stack(data))
else:
# still construct a table, but with no rows
labels = ["channel", "frequency", "pvcorr"]
units = ["number", "GHz", "N/A"]
table = Table(columns=labels,units=units)
b3.setkey("table",table)
b3.setkey("sigma",float(rms))
dt.tag("table")
if len(corr) > 0:
table.exportTable(self.dir("testPVCorr.tab"),cols=['frequency','pvcorr'])
test_single(ch,freq,corr)
logging.regression("PVC: %f %f" % (corr.min(),corr.max()))
title = 'PVCorr mode=%d [%d,%d] %g' % (mode,ch0,ch1,cutoff)
x = ch
xlab = 'Channel'
y = [corr]
ylab = 'PV Correlation'
p1 = "%s_%d" % (bdp_name,0)
segp = []
segp.append( [0,len(ch),0.0,0.0] )
segp.append( [0,len(ch),3.0*rms, 3.0*rms] )
# @todo: in principle we know with given noise and size of box, what the sigma in pvcorr should be
self.myplot.plotter(x,y,title,figname=p1,xlab=xlab,ylab=ylab,segments=segp, thumbnail=True)
#out1 = np.rot90 (data.reshape((nvel,npos)) )
if mode > 1:
self.myplot.map1(data=out,title="testing PVCorr_AT: mode%d"%mode,figname='testPVCorr', thumbnail=True)
taskargs = "numsigma=%.1f range=[%d,%d]" % (numsigma,ch0,ch1)
caption = "Position-velocity correlation plot"
thumbname = self.myplot.getThumbnail(figno=self.myplot.figno,relative=True)
figname = self.myplot.getFigure(figno=self.myplot.figno,relative=True)
image = Image(images={bt.PNG: figname}, thumbnail=thumbname, thumbnailtype=bt.PNG,
description=caption)
b3.image.addimage(image, "pvcorr")
self._summary["pvcorr"] = SummaryEntry([figname,thumbname,caption,fin],"PVCorr_AT",self.id(True),taskargs)
else:
self._summary["pvcorr"] = None
logging.warning("No summary")
logging.regression("PVC: -1")
dt.tag("done")
dt.end()