bandNum = int(band.strip('B')) if band not in bands.keys(): for j in xrange(bandNum,0,-1): newBand = 'B' + '%03d' % j if newBand in bands.keys(): bandsToUse[i] = newBand break for band in bands.keys(): if band not in bandsToUse: del bands[band] # This can cause a failure if an alternate band selected above gets "removed" # but calling this step before the serach for RGB above leads to unexpected # bad behavior imageList = utilities.preprocessImage(bands, multipliers, wavelengths, {}) imageArray = numpy.array(imageList) numpy.exp(imageArray,imageArray) imageRGB = numpy.zeros(mask.shape + (3,)) rgbMin = imageArray.min() rgbMax = imageArray.max() colorList = ["#ff0087","#00ff78","#7700ff","#ffb2f4","#ffff00","#00ff00","#ff00ff","#ff0000","#ff6633","#66ff00"] for i in numpy.arange(nx.size): imageRGB[ny[i],nx[i],0] = 255.*(imageArray[0,i] - rgbMin)/(rgbMax - rgbMin) imageRGB[ny[i],nx[i],1] = 255.*(imageArray[1,i] - rgbMin)/(rgbMax - rgbMin)
imageData["numPixels"] = numpy.nonzero(mask)[0].size else: bands[key] = numpy.array(value[mask],dtype=numpy.float64) if imageData["metadata"] is not None: if 'HSI' in imageData["metadata"].keys(): wavelengths = {} multipliers = {} for w,wave in enumerate(imageData["metadata"][unicode("HSI")][unicode("wavelength")]): wavelengths["B" + "%03d" % w] = float(wave) multipliers["B" + "%03d" % w] = 1 else: wavelengths = imageData["metadata"]["bandWavelength"] multipliers = imageData["metadata"]["bandMultiplier"] imageList = utilities.preprocessImage(bands, multipliers, wavelengths, imageData) sys.stderr.write("This is the number of bands: %r\n" % len(imageList)) iDot,iPartial = main(imageList) imageData["imageDot"] = iDot imageData["imagePartial"] = iPartial if noiseFlag.upper() == "TRUE": noiseList = calcNoise(imageList,mask) nDot,nPartial = main(noiseList) imageData["noiseDot"] = nDot imageData["noisePartial"] = nPartial try: regionKey = imageData["metadata"]["originalDirName"] except KeyError: regionKey = imageData["metadata"]["outputFile"]
def process(self, tup): localFileName = tup.values[0] hdfsFileName = tup.values[1] imageData = {} imageData["metadata"] = None storm.log("start processing %s %s" % (localFileName, hdfsFileName)) for key, sorter, interpretedValue in binaryhadoop.readFromHDFSiter(hdfsFileName): if key == "metadata": imageData["metadata"] = interpretedValue bands = {} storm.log(" read metadata") elif key == "mask": mask = utilities.rollMask(interpretedValue > 0) numPixels = numpy.nonzero(mask)[0].size storm.log(" read mask") else: bands[key] = interpretedValue[mask] storm.log(" read band %s" % key) if imageData["metadata"] is not None: wavelengths = imageData["metadata"]["bandWavelength"] multipliers = imageData["metadata"]["bandMultiplier"] storm.log("making imageArray 1") imageList = utilities.preprocessImage(bands, multipliers, wavelengths, imageData) #find the covariance of the image bands storm.log("making covariance") imageCov = self.makeCovariance(imageList, numPixels) #find the principal components (eigenvectors/values) storm.log("making principle components 1") imgV, imgP = numpy.linalg.eig(imageCov) # storm.log("making principle components 2") indexList = numpy.argsort(-imgV) imgV = imgV[indexList] imgP = imgP[:,indexList] storm.log("making variance percentage") xVarianceComponents = 5 variancePercentage = [x/numpy.sum(imgV) for x in imgV][:xVarianceComponents] storm.log("making rogue bands") rogueBands = self.checkpca(imgP.T,xVarianceComponents) storm.log("making gray bands") bandGray = numpy.zeros(len(imageList[0])) for band in imageList: bandGray += (numpy.array(band))**2 bandGray = numpy.sort(bandGray) bandPercent = 1 #The 99th percentile is removed to avoid skewing the mean bandGray = bandGray[bandGray < numpy.percentile(bandGray,100-bandPercent)] #Histogram is created storm.log("making gray band histogram") [hist,bin_edges] = numpy.histogram(bandGray,bins=100) #Locate the peaks on the histogram storm.log("making peaks and valleys") peaks, valleys = self.findpeaks(hist,3,(bin_edges[:-1] + bin_edges[1:])/2) #Find mean and standard deviation of all pixels bandMean = numpy.mean(bandGray) bandSigma = numpy.std(bandGray) storm.log("making JSON output") imageData["numPixels"] = int(numPixels) imageData["grayBandMean"] = float(bandMean) imageData["grayBandSigma"] = float(bandSigma) #Report percentage of total pixels which lie beyond one standard deviation from mean imageData["grayBandPlusOneSigma"] = float(numpy.sum(bandGray > (bandMean+bandSigma))/numpy.float(numPixels)) imageData["grayBandMinusOneSigma"] = float(numpy.sum(bandGray < (bandMean-bandSigma))/numpy.float(numPixels)) imageData["grayBandHistPeaks"] = [[float(x), int(y)] for x, y in peaks] #imageData["grayBandHistValleys"] = [[float(x), int(y)] for x, y in valleys] #PCA analysis and sum of first 5 principal components imageData["grayBandExplainedVariance"] = [float(x) for x in variancePercentage] #Report bands that have high leave-one-out loading variance imageData["grayBandRogueBands"] = [str(x) for x in rogueBands] #Report histogram imageData["grayBandHistogram"] = [[float(x) for x in bin_edges], [int(x) for x in hist]] #emit the final statistics storm.log("emiting Storm tuple") storm.emit([localFileName, hdfsFileName, json.dumps(imageData)], stream="summaryStatistics") storm.log("done with %s %s" % (localFileName, hdfsFileName))
mask = utilities.fixMask(mask,bands) for bandKey, bandValue in bands.iteritems(): bands[bandKey] = bandValue[mask] if 'HSI' in imageData["metadata"].keys(): wavelengths = {} multipliers = {} for w,wave in enumerate(imageData["metadata"][unicode("HSI")][unicode("wavelength")]): wavelengths["B" + "%03d" % w] = float(wave) multipliers["B" + "%03d" % w] = 1 else: wavelengths = imageData["metadata"]["bandWavelength"] multipliers = imageData["metadata"]["bandMultiplier"] imageList = utilities.preprocessImage(bands, multipliers, wavelengths, imageData, selectBands=selectbands) sys.stderr.write("This is the number of bands: %r\n" % len(imageList)) iDot,iPartial = main(imageList) imageData["imageDot"] = iDot imageData["imagePartial"] = iPartial if noiseFlag.upper() == "TRUE": noiseList = calcNoise(imageList,mask) nDot,nPartial = main(noiseList) imageData["noiseDot"] = nDot imageData["noisePartial"] = nPartial try: regionKey = imageData["metadata"]["originalDirName"] except KeyError: regionKey = imageData["metadata"]["outputFile"]