def testSerialization(self): self.image.images = { admit.bdp_types.FITS: "myimage.fits", admit.bdp_types.CASA: "myimage.casa" } self.image.thumbnail = "myimage_thumb.png" self.image.thumbnailtype = admit.bdp_types.PNG self.image.auxiliary = "myaux.jpg" self.image.auxtype = admit.bdp_types.JPG self.image.description = "This is a image for serialization unit test" self.image.name = "My name is legion" out = self.image.serialize() myImage = admit.Image() myImage.deserialize(out) self.assertEqual(self.image, myImage) self.image.description = "Bad wolf" self.assertNotEqual(self.image, myImage) myImage2 = admit.Image() myImage2.deserialize(out) multi_image = admit.MultiImage() myImage.description = "First image" multi_image.addimage(myImage, "first") myImage2.description = "Second image" multi_image.addimage(myImage2, "second") out = multi_image.serialize() multi_image2 = admit.MultiImage() multi_image2.deserialize(out) out2 = multi_image2.serialize() self.assertEqual(out, out2) self.assertEqual(multi_image, multi_image2)
def run(self): """ Task run method. Computes the principal component decomposition of the input images and populates the output eigenimages and projection matrix. Parameters ---------- None Returns ------- None """ self._summary = {} # BDP output uses the alias name if provided, else a flow-unique one. stem = self._alias if not stem: stem = "pca%d" % (self.id()) inum = 0 data = [] icols = [] for ibdp in self._bdp_in: # Convert input CASA images to numpy arrays. istem = ibdp.getimagefile(bt.CASA) ifile = ibdp.baseDir() + istem icols.append(os.path.splitext(istem)[0]) if os.path.dirname(icols[-1]): icols[-1] = os.path.dirname(icols[-1]) # Typical line cube case. img = admit.casautil.getdata(ifile, zeromask=True).data data.append(img) admit.logging.info("%s shape=%s min=%g max=%g" % (icols[-1], str(img.shape), np.amin(img), np.amax(img))) assert len(data[0].shape) == 2, "Only 2-D input images supported" assert data[0].shape == data[inum].shape, "Input shapes must match" inum += 1 # At least two inputs required for meaningful PCA! assert inum >= 2, "At least two input images required" # Each 2-D input image is a plane in a single multi-color image. # Each color multiplet (one per pixel) is an observation. # For PCA we collate the input images into a vector of observations. shape = data[0].shape npix = shape[0] * shape[1] clip = self.getkey('clipvals') if not clip: clip = [0 for i in range(inum)] assert len(clip) >= inum, "Too few clipvals provided" # Clip input values and stack into a vector of observations. pca_data = [] for i in range(inum): nd = data[i] nd[nd < clip[i]] = 0.0 pca_data.append(np.reshape(nd, (npix,1))) pca_in = np.hstack(pca_data) pca = mlab.PCA(pca_in) # Input statistics and output variance fractions. #print "fracs:", pca.fracs #print "mean:", pca.mu #print "sdev:", pca.sigma obdp = admit.Table_BDP(stem+"_stats") obdp.table.setData(np.vstack([pca.mu, pca.sigma,pca.fracs]).T) obdp.table.columns = ["Input mean", "Input deviation", "Eigenimage variance fraction"] obdp.table.description = "PCA Image Statistics" self.addoutput(obdp) # Pre-format columns for summary output. # This is required when mixing strings and numbers in a table. # (NumPy will output the array as all strings.) table1 = admit.Table() table1.setData(np.vstack([[i for i in range(inum)], icols, ["%.3e" % x for x in pca.mu], ["%.3e" % x for x in pca.sigma], ["%s_eigen/%d.im" % (stem, i) for i in range(inum)], ["%.4f" % x for x in pca.fracs]]).T) table1.columns = ["Index", "Input", "Input mean", "Input deviation", "Eigenimage", "Eigenimage variance fraction"] table1.description = "PCA Image Statistics" # Projection matrix (eigenvectors). #print "projection:", pca.Wt obdp = admit.Table_BDP(stem + "_proj") obdp.table.setData(pca.Wt) obdp.table.columns = icols obdp.table.description = \ "PCA Projection Matrix (normalized input to output)" self.addoutput(obdp) # Covariance matrix. covar = np.cov(pca.a, rowvar=0, bias=1) #print "covariance:", covar obdp = admit.Table_BDP(stem + "_covar") obdp.table.setData(covar) obdp.table.columns = icols obdp.table.description = "PCA Covariance Matrix" self.addoutput(obdp) # Collate projected observations into eigenimages and save output. os.mkdir(self.baseDir()+stem+"_eigen") pca_out = np.hsplit(pca.Y, inum) odata = [] for i in range(inum): ofile = "%s_eigen/%d" % (stem, i) img = np.reshape(pca_out[i], shape) odata.append(img) #print ofile, "shape, min, max:", img.shape, np.amin(img), np.amax(img) aplot = admit.util.APlot(figno=inum, abspath=self.baseDir(), ptype=admit.util.PlotControl.PNG) aplot.map1(np.rot90(img), title=ofile, figname=ofile) aplot.final() # Currently the output eigenimages are stored as PNG files only. admit.casautil.putdata_raw(self.baseDir()+ofile+".im", img, ifile) oimg = admit.Image() oimg.addimage(admit.imagedescriptor(ofile+".im", format=bt.CASA)) oimg.addimage(admit.imagedescriptor(ofile+".png", format=bt.PNG)) obdp = admit.Image_BDP(ofile) obdp.addimage(oimg) self.addoutput(obdp) # As a cross-check, reconstruct input images and compute differences. for k in range(inum): ximg = pca.Wt[0][k]*odata[0] for l in range(1,inum): ximg += pca.Wt[l][k]*odata[l] ximg = pca.mu[k] + pca.sigma[k]*ximg admit.logging.regression("PCA: %s residual: " % icols[k] + str(np.linalg.norm(ximg - data[k]))) # Collect large covariance values for summary. cvmin = self.getkey('covarmin') cvsum = [] cvmax = 0.0 for i in range(inum): for j in range(i+1, inum): if abs(covar[i][j]) >= cvmax: cvmax = abs(covar[i][j]) if abs(covar[i][j]) >= cvmin: cvsum.append([icols[i], icols[j], "%.4f" % (covar[i][j])]) admit.logging.regression("PCA: Covariances > %.4f: %s (max: %.4f)" % (cvmin,str(cvsum),cvmax)) table2 = admit.Table() table2.columns = ["Input1", "Input2", "Covariance"] table2.setData(cvsum) table2.description = "PCA High Covariance Summary" keys = "covarmin=%.4f clipvals=%s" % (cvmin, str(clip)) self._summary["pca"] = admit.SummaryEntry([table1.serialize(), table2.serialize() ], "PrincipalComponent_AT", self.id(True), keys)
def setUp(self): self.verbose = False self.testName = "Utility Image Class Unit Test" self.image = admit.Image()
def run(self): """ Task run method. Outputs three sample data products from a single input data cube: 1. A cube scaled by the value of *cubescale*. 2. An image sliced from the cube at frequency *imgfreq*. 3. A spectrum extracted from the cube at position *specpos*. Parameters ---------- None Returns ------- None """ self._summary = {} # BDP output uses the alias name if provided, else a flow-unique name. stem = self._alias if not stem: stem = "template%d" % (self.id()) # The input data cube BDP. ibdp = self._bdp_in[0] # The data cube is a CASA image on disk. # Get its file name, read it in and convert to a NumPy array. # The entire cube is read into memory (large cubes may exceed RAM!) # Any masked pixels are converted to zero. istem = ibdp.getimagefile(bt.CASA) ifile = ibdp.baseDir() + istem icube = admit.casautil.getdata(ifile, zeromask=True).data assert len(icube.shape) == 3, "Only 3-D data cubes supported" admit.logging.info((istem + " dimensions: (%d, %d, %d)") % icube.shape) # =================================================================== # 1. Scale the entire cube by 'cubescale'. # =================================================================== cubescale = self.getkey('cubescale') ocube = icube * cubescale ocubestem = "%s_cube" % stem ocubefile = self.baseDir() + ocubestem # # Create a CASA ouput image (no mask information). admit.casautil.putdata_raw(ocubefile + ".im", ocube, ifile) # # Output the result as a new SpwCube. image = admit.Image(description="Template 3-D Cube") image.addimage(admit.imagedescriptor(ocubefile + ".im", format=bt.CASA)) obdp1 = admit.SpwCube_BDP(ocubestem) obdp1.addimage(image) self.addoutput(obdp1) # =================================================================== # 2. Slice a 2-D image from the cube at frequency 'imgslice'. # =================================================================== # # The frequency axis is the third array index. # Clip slice to be in range. imgslice = max(0, min(ocube.shape[2] - 1, self.getkey('imgslice'))) oimg = ocube[:, :, imgslice] oimgstem = "%s_img" % stem oimgfile = self.baseDir() + oimgstem # # Create a CASA format ouput image. # @TODO Does not work for 3-D -> 2-D output. #admit.casautil.putdata_raw(oimgfile+".im", oimg, ifile) # # Create a PNG format output image. # Image data must be rotated to match matplotlib axis order. oimgtitle = "Template Image Slice @ channel %d" % imgslice aplot = admit.APlot(figno=1, abspath=self.baseDir(), pmode=admit.PlotControl.BATCH, ptype=admit.PlotControl.PNG) aplot.map1(np.rot90(oimg), xlab="R.A. (pixels)", ylab="Dec (pixels)", title=oimgtitle, figname=oimgstem) aplot.final() # # Output data product referencing both image formats. image = admit.Image(description="Template 2-D Image") image.addimage(admit.imagedescriptor(oimgfile + ".im", format=bt.CASA)) image.addimage(admit.imagedescriptor(oimgstem + ".png", format=bt.PNG)) obdp2 = admit.Image_BDP(oimgstem) obdp2.addimage(image) self.addoutput(obdp2) # =================================================================== # 3. Extract a 1-D spectrum from the cube at position 'specpos'. # =================================================================== # # Clip position to be in range. specpos = self.getkey('specpos') specpos = (max(0, min(ocube.shape[0] - 1, specpos[0])), max(0, min(ocube.shape[1] - 1, specpos[1]))) ospec = ocube[specpos[0], specpos[1], :] ochan = np.arange(ospec.shape[0]) ospecstem = "%s_spec" % stem # # Create a PNG plot (standalone). ospectitle = "Template Spectrum @ position %s" % str(specpos) aplot = admit.APlot(figno=2, abspath=self.baseDir(), pmode=admit.PlotControl.BATCH, ptype=admit.PlotControl.PNG) aplot.plotter(x=ochan, y=[ospec], xlab="Channel", ylab="Intensity", title=ospectitle, figname=ospecstem) aplot.final() # # Output data product (spectrum table). obdp3 = admit.Table_BDP(ospecstem) obdp3.table.description = "Template 1-D Spectrum" obdp3.table.columns = ["Channel", "Spectrum @ (%d, %d)" % specpos] obdp3.table.setData(np.transpose(np.vstack([ochan, ospec]))) self.addoutput(obdp3) # =================================================================== # Populate example summary object. # =================================================================== # # NOTE: Summary.py, etc., must be updated to recognize this and # for Template_AT to appear in the HTML summary. stats = admit.Table() stats.description = "Template Data Product Statistics" stats.columns = ["Statistic", "InCube", "OutCube", "Image", "Spectrum"] stats.setData( np.array([[ "Min", np.amin(icube), np.amin(ocube), np.amin(oimg), np.amin(ospec) ], [ "Max", np.amax(icube), np.amax(ocube), np.amax(oimg), np.amax(ospec) ]])) # # Data product statistics table. keys = "cubescale=%s imgslice=%d specpos=%s" % \ (str(cubescale), imgslice, str(specpos)) self._summary["template"] = admit.SummaryEntry([stats.serialize()], "Template_AT", self.id(True), keys) # # Plots for the image and spectrum. ocubeim = ocubestem + ".im" spec_description = [[ 0, 0, "", "", oimgstem + ".png", oimgstem + "_thumb.png", oimgtitle, ocubeim ], [ specpos[0], specpos[1], "", "Channel", ospecstem + ".png", ospecstem + "_thumb.png", ospectitle, ocubeim ]] self._summary["spectra"] = admit.SummaryEntry(spec_description, "Template_AT", self.id(True), keys)