def testSerialization(self):
self.image.images = {
admit.bdp_types.FITS: "myimage.fits",
admit.bdp_types.CASA: "myimage.casa"
}
self.image.thumbnail = "myimage_thumb.png"
self.image.thumbnailtype = admit.bdp_types.PNG
self.image.auxiliary = "myaux.jpg"
self.image.auxtype = admit.bdp_types.JPG
self.image.description = "This is a image for serialization unit test"
self.image.name = "My name is legion"
out = self.image.serialize()
myImage = admit.Image()
myImage.deserialize(out)
self.assertEqual(self.image, myImage)
self.image.description = "Bad wolf"
self.assertNotEqual(self.image, myImage)
myImage2 = admit.Image()
myImage2.deserialize(out)
multi_image = admit.MultiImage()
myImage.description = "First image"
multi_image.addimage(myImage, "first")
myImage2.description = "Second image"
multi_image.addimage(myImage2, "second")
out = multi_image.serialize()
multi_image2 = admit.MultiImage()
multi_image2.deserialize(out)
out2 = multi_image2.serialize()
self.assertEqual(out, out2)
self.assertEqual(multi_image, multi_image2)
def run(self):
"""
Task run method.
Computes the principal component decomposition of the input images and
populates the output eigenimages and projection matrix.
Parameters
----------
None
Returns
-------
None
"""
self._summary = {}
# BDP output uses the alias name if provided, else a flow-unique one.
stem = self._alias
if not stem: stem = "pca%d" % (self.id())
inum = 0
data = []
icols = []
for ibdp in self._bdp_in:
# Convert input CASA images to numpy arrays.
istem = ibdp.getimagefile(bt.CASA)
ifile = ibdp.baseDir() + istem
icols.append(os.path.splitext(istem)[0])
if os.path.dirname(icols[-1]):
icols[-1] = os.path.dirname(icols[-1]) # Typical line cube case.
img = admit.casautil.getdata(ifile, zeromask=True).data
data.append(img)
admit.logging.info("%s shape=%s min=%g max=%g" %
(icols[-1], str(img.shape), np.amin(img), np.amax(img)))
assert len(data[0].shape) == 2, "Only 2-D input images supported"
assert data[0].shape == data[inum].shape, "Input shapes must match"
inum += 1
# At least two inputs required for meaningful PCA!
assert inum >= 2, "At least two input images required"
# Each 2-D input image is a plane in a single multi-color image.
# Each color multiplet (one per pixel) is an observation.
# For PCA we collate the input images into a vector of observations.
shape = data[0].shape
npix = shape[0] * shape[1]
clip = self.getkey('clipvals')
if not clip: clip = [0 for i in range(inum)]
assert len(clip) >= inum, "Too few clipvals provided"
# Clip input values and stack into a vector of observations.
pca_data = []
for i in range(inum):
nd = data[i]
nd[nd < clip[i]] = 0.0
pca_data.append(np.reshape(nd, (npix,1)))
pca_in = np.hstack(pca_data)
pca = mlab.PCA(pca_in)
# Input statistics and output variance fractions.
#print "fracs:", pca.fracs
#print "mean:", pca.mu
#print "sdev:", pca.sigma
obdp = admit.Table_BDP(stem+"_stats")
obdp.table.setData(np.vstack([pca.mu, pca.sigma,pca.fracs]).T)
obdp.table.columns = ["Input mean", "Input deviation",
"Eigenimage variance fraction"]
obdp.table.description = "PCA Image Statistics"
self.addoutput(obdp)
# Pre-format columns for summary output.
# This is required when mixing strings and numbers in a table.
# (NumPy will output the array as all strings.)
table1 = admit.Table()
table1.setData(np.vstack([[i for i in range(inum)],
icols,
["%.3e" % x for x in pca.mu],
["%.3e" % x for x in pca.sigma],
["%s_eigen/%d.im" % (stem, i)
for i in range(inum)],
["%.4f" % x for x in pca.fracs]]).T)
table1.columns = ["Index", "Input", "Input mean",
"Input deviation",
"Eigenimage",
"Eigenimage variance fraction"]
table1.description = "PCA Image Statistics"
# Projection matrix (eigenvectors).
#print "projection:", pca.Wt
obdp = admit.Table_BDP(stem + "_proj")
obdp.table.setData(pca.Wt)
obdp.table.columns = icols
obdp.table.description = \
"PCA Projection Matrix (normalized input to output)"
self.addoutput(obdp)
# Covariance matrix.
covar = np.cov(pca.a, rowvar=0, bias=1)
#print "covariance:", covar
obdp = admit.Table_BDP(stem + "_covar")
obdp.table.setData(covar)
obdp.table.columns = icols
obdp.table.description = "PCA Covariance Matrix"
self.addoutput(obdp)
# Collate projected observations into eigenimages and save output.
os.mkdir(self.baseDir()+stem+"_eigen")
pca_out = np.hsplit(pca.Y, inum)
odata = []
for i in range(inum):
ofile = "%s_eigen/%d" % (stem, i)
img = np.reshape(pca_out[i], shape)
odata.append(img)
#print ofile, "shape, min, max:", img.shape, np.amin(img), np.amax(img)
aplot = admit.util.APlot(figno=inum, abspath=self.baseDir(),
ptype=admit.util.PlotControl.PNG)
aplot.map1(np.rot90(img), title=ofile, figname=ofile)
aplot.final()
# Currently the output eigenimages are stored as PNG files only.
admit.casautil.putdata_raw(self.baseDir()+ofile+".im", img, ifile)
oimg = admit.Image()
oimg.addimage(admit.imagedescriptor(ofile+".im", format=bt.CASA))
oimg.addimage(admit.imagedescriptor(ofile+".png", format=bt.PNG))
obdp = admit.Image_BDP(ofile)
obdp.addimage(oimg)
self.addoutput(obdp)
# As a cross-check, reconstruct input images and compute differences.
for k in range(inum):
ximg = pca.Wt[0][k]*odata[0]
for l in range(1,inum):
ximg += pca.Wt[l][k]*odata[l]
ximg = pca.mu[k] + pca.sigma[k]*ximg
admit.logging.regression("PCA: %s residual: " % icols[k] +
str(np.linalg.norm(ximg - data[k])))
# Collect large covariance values for summary.
cvmin = self.getkey('covarmin')
cvsum = []
cvmax = 0.0
for i in range(inum):
for j in range(i+1, inum):
if abs(covar[i][j]) >= cvmax:
cvmax = abs(covar[i][j])
if abs(covar[i][j]) >= cvmin:
cvsum.append([icols[i], icols[j], "%.4f" % (covar[i][j])])
admit.logging.regression("PCA: Covariances > %.4f: %s (max: %.4f)" % (cvmin,str(cvsum),cvmax))
table2 = admit.Table()
table2.columns = ["Input1", "Input2", "Covariance"]
table2.setData(cvsum)
table2.description = "PCA High Covariance Summary"
keys = "covarmin=%.4f clipvals=%s" % (cvmin, str(clip))
self._summary["pca"] = admit.SummaryEntry([table1.serialize(),
table2.serialize()
],
"PrincipalComponent_AT",
self.id(True), keys)
def setUp(self):
self.verbose = False
self.testName = "Utility Image Class Unit Test"
self.image = admit.Image()
def run(self):
"""
Task run method.
Outputs three sample data products from a single input data cube:
1. A cube scaled by the value of *cubescale*.
2. An image sliced from the cube at frequency *imgfreq*.
3. A spectrum extracted from the cube at position *specpos*.
Parameters
----------
None
Returns
-------
None
"""
self._summary = {}
# BDP output uses the alias name if provided, else a flow-unique name.
stem = self._alias
if not stem: stem = "template%d" % (self.id())
# The input data cube BDP.
ibdp = self._bdp_in[0]
# The data cube is a CASA image on disk.
# Get its file name, read it in and convert to a NumPy array.
# The entire cube is read into memory (large cubes may exceed RAM!)
# Any masked pixels are converted to zero.
istem = ibdp.getimagefile(bt.CASA)
ifile = ibdp.baseDir() + istem
icube = admit.casautil.getdata(ifile, zeromask=True).data
assert len(icube.shape) == 3, "Only 3-D data cubes supported"
admit.logging.info((istem + " dimensions: (%d, %d, %d)") % icube.shape)
# ===================================================================
# 1. Scale the entire cube by 'cubescale'.
# ===================================================================
cubescale = self.getkey('cubescale')
ocube = icube * cubescale
ocubestem = "%s_cube" % stem
ocubefile = self.baseDir() + ocubestem
#
# Create a CASA ouput image (no mask information).
admit.casautil.putdata_raw(ocubefile + ".im", ocube, ifile)
#
# Output the result as a new SpwCube.
image = admit.Image(description="Template 3-D Cube")
image.addimage(admit.imagedescriptor(ocubefile + ".im",
format=bt.CASA))
obdp1 = admit.SpwCube_BDP(ocubestem)
obdp1.addimage(image)
self.addoutput(obdp1)
# ===================================================================
# 2. Slice a 2-D image from the cube at frequency 'imgslice'.
# ===================================================================
#
# The frequency axis is the third array index.
# Clip slice to be in range.
imgslice = max(0, min(ocube.shape[2] - 1, self.getkey('imgslice')))
oimg = ocube[:, :, imgslice]
oimgstem = "%s_img" % stem
oimgfile = self.baseDir() + oimgstem
#
# Create a CASA format ouput image.
# @TODO Does not work for 3-D -> 2-D output.
#admit.casautil.putdata_raw(oimgfile+".im", oimg, ifile)
#
# Create a PNG format output image.
# Image data must be rotated to match matplotlib axis order.
oimgtitle = "Template Image Slice @ channel %d" % imgslice
aplot = admit.APlot(figno=1,
abspath=self.baseDir(),
pmode=admit.PlotControl.BATCH,
ptype=admit.PlotControl.PNG)
aplot.map1(np.rot90(oimg),
xlab="R.A. (pixels)",
ylab="Dec (pixels)",
title=oimgtitle,
figname=oimgstem)
aplot.final()
#
# Output data product referencing both image formats.
image = admit.Image(description="Template 2-D Image")
image.addimage(admit.imagedescriptor(oimgfile + ".im", format=bt.CASA))
image.addimage(admit.imagedescriptor(oimgstem + ".png", format=bt.PNG))
obdp2 = admit.Image_BDP(oimgstem)
obdp2.addimage(image)
self.addoutput(obdp2)
# ===================================================================
# 3. Extract a 1-D spectrum from the cube at position 'specpos'.
# ===================================================================
#
# Clip position to be in range.
specpos = self.getkey('specpos')
specpos = (max(0, min(ocube.shape[0] - 1, specpos[0])),
max(0, min(ocube.shape[1] - 1, specpos[1])))
ospec = ocube[specpos[0], specpos[1], :]
ochan = np.arange(ospec.shape[0])
ospecstem = "%s_spec" % stem
#
# Create a PNG plot (standalone).
ospectitle = "Template Spectrum @ position %s" % str(specpos)
aplot = admit.APlot(figno=2,
abspath=self.baseDir(),
pmode=admit.PlotControl.BATCH,
ptype=admit.PlotControl.PNG)
aplot.plotter(x=ochan,
y=[ospec],
xlab="Channel",
ylab="Intensity",
title=ospectitle,
figname=ospecstem)
aplot.final()
#
# Output data product (spectrum table).
obdp3 = admit.Table_BDP(ospecstem)
obdp3.table.description = "Template 1-D Spectrum"
obdp3.table.columns = ["Channel", "Spectrum @ (%d, %d)" % specpos]
obdp3.table.setData(np.transpose(np.vstack([ochan, ospec])))
self.addoutput(obdp3)
# ===================================================================
# Populate example summary object.
# ===================================================================
#
# NOTE: Summary.py, etc., must be updated to recognize this and
# for Template_AT to appear in the HTML summary.
stats = admit.Table()
stats.description = "Template Data Product Statistics"
stats.columns = ["Statistic", "InCube", "OutCube", "Image", "Spectrum"]
stats.setData(
np.array([[
"Min",
np.amin(icube),
np.amin(ocube),
np.amin(oimg),
np.amin(ospec)
],
[
"Max",
np.amax(icube),
np.amax(ocube),
np.amax(oimg),
np.amax(ospec)
]]))
#
# Data product statistics table.
keys = "cubescale=%s imgslice=%d specpos=%s" % \
(str(cubescale), imgslice, str(specpos))
self._summary["template"] = admit.SummaryEntry([stats.serialize()],
"Template_AT",
self.id(True), keys)
#
# Plots for the image and spectrum.
ocubeim = ocubestem + ".im"
spec_description = [[
0, 0, "", "", oimgstem + ".png", oimgstem + "_thumb.png",
oimgtitle, ocubeim
],
[
specpos[0], specpos[1], "", "Channel",
ospecstem + ".png", ospecstem + "_thumb.png",
ospectitle, ocubeim
]]
self._summary["spectra"] = admit.SummaryEntry(spec_description,
"Template_AT",
self.id(True), keys)