bandNum = int(band.strip('B'))
if band not in bands.keys():
for j in xrange(bandNum,0,-1):
newBand = 'B' + '%03d' % j
if newBand in bands.keys():
bandsToUse[i] = newBand
break
for band in bands.keys():
if band not in bandsToUse:
del bands[band]
# This can cause a failure if an alternate band selected above gets "removed"
# but calling this step before the serach for RGB above leads to unexpected
# bad behavior
imageList = utilities.preprocessImage(bands, multipliers, wavelengths, {})
imageArray = numpy.array(imageList)
numpy.exp(imageArray,imageArray)
imageRGB = numpy.zeros(mask.shape + (3,))
rgbMin = imageArray.min()
rgbMax = imageArray.max()
colorList = ["#ff0087","#00ff78","#7700ff","#ffb2f4","#ffff00","#00ff00","#ff00ff","#ff0000","#ff6633","#66ff00"]
for i in numpy.arange(nx.size):
imageRGB[ny[i],nx[i],0] = 255.*(imageArray[0,i] - rgbMin)/(rgbMax - rgbMin)
imageRGB[ny[i],nx[i],1] = 255.*(imageArray[1,i] - rgbMin)/(rgbMax - rgbMin)
imageData["numPixels"] = numpy.nonzero(mask)[0].size
else:
bands[key] = numpy.array(value[mask],dtype=numpy.float64)
if imageData["metadata"] is not None:
if 'HSI' in imageData["metadata"].keys():
wavelengths = {}
multipliers = {}
for w,wave in enumerate(imageData["metadata"][unicode("HSI")][unicode("wavelength")]):
wavelengths["B" + "%03d" % w] = float(wave)
multipliers["B" + "%03d" % w] = 1
else:
wavelengths = imageData["metadata"]["bandWavelength"]
multipliers = imageData["metadata"]["bandMultiplier"]
imageList = utilities.preprocessImage(bands, multipliers, wavelengths, imageData)
sys.stderr.write("This is the number of bands: %r\n" % len(imageList))
iDot,iPartial = main(imageList)
imageData["imageDot"] = iDot
imageData["imagePartial"] = iPartial
if noiseFlag.upper() == "TRUE":
noiseList = calcNoise(imageList,mask)
nDot,nPartial = main(noiseList)
imageData["noiseDot"] = nDot
imageData["noisePartial"] = nPartial
try:
regionKey = imageData["metadata"]["originalDirName"]
except KeyError:
regionKey = imageData["metadata"]["outputFile"]
def process(self, tup):
localFileName = tup.values[0]
hdfsFileName = tup.values[1]
imageData = {}
imageData["metadata"] = None
storm.log("start processing %s %s" % (localFileName, hdfsFileName))
for key, sorter, interpretedValue in binaryhadoop.readFromHDFSiter(hdfsFileName):
if key == "metadata":
imageData["metadata"] = interpretedValue
bands = {}
storm.log(" read metadata")
elif key == "mask":
mask = utilities.rollMask(interpretedValue > 0)
numPixels = numpy.nonzero(mask)[0].size
storm.log(" read mask")
else:
bands[key] = interpretedValue[mask]
storm.log(" read band %s" % key)
if imageData["metadata"] is not None:
wavelengths = imageData["metadata"]["bandWavelength"]
multipliers = imageData["metadata"]["bandMultiplier"]
storm.log("making imageArray 1")
imageList = utilities.preprocessImage(bands, multipliers, wavelengths, imageData)
#find the covariance of the image bands
storm.log("making covariance")
imageCov = self.makeCovariance(imageList, numPixels)
#find the principal components (eigenvectors/values)
storm.log("making principle components 1")
imgV, imgP = numpy.linalg.eig(imageCov)
#
storm.log("making principle components 2")
indexList = numpy.argsort(-imgV)
imgV = imgV[indexList]
imgP = imgP[:,indexList]
storm.log("making variance percentage")
xVarianceComponents = 5
variancePercentage = [x/numpy.sum(imgV) for x in imgV][:xVarianceComponents]
storm.log("making rogue bands")
rogueBands = self.checkpca(imgP.T,xVarianceComponents)
storm.log("making gray bands")
bandGray = numpy.zeros(len(imageList[0]))
for band in imageList:
bandGray += (numpy.array(band))**2
bandGray = numpy.sort(bandGray)
bandPercent = 1
#The 99th percentile is removed to avoid skewing the mean
bandGray = bandGray[bandGray < numpy.percentile(bandGray,100-bandPercent)]
#Histogram is created
storm.log("making gray band histogram")
[hist,bin_edges] = numpy.histogram(bandGray,bins=100)
#Locate the peaks on the histogram
storm.log("making peaks and valleys")
peaks, valleys = self.findpeaks(hist,3,(bin_edges[:-1] + bin_edges[1:])/2)
#Find mean and standard deviation of all pixels
bandMean = numpy.mean(bandGray)
bandSigma = numpy.std(bandGray)
storm.log("making JSON output")
imageData["numPixels"] = int(numPixels)
imageData["grayBandMean"] = float(bandMean)
imageData["grayBandSigma"] = float(bandSigma)
#Report percentage of total pixels which lie beyond one standard deviation from mean
imageData["grayBandPlusOneSigma"] = float(numpy.sum(bandGray > (bandMean+bandSigma))/numpy.float(numPixels))
imageData["grayBandMinusOneSigma"] = float(numpy.sum(bandGray < (bandMean-bandSigma))/numpy.float(numPixels))
imageData["grayBandHistPeaks"] = [[float(x), int(y)] for x, y in peaks]
#imageData["grayBandHistValleys"] = [[float(x), int(y)] for x, y in valleys]
#PCA analysis and sum of first 5 principal components
imageData["grayBandExplainedVariance"] = [float(x) for x in variancePercentage]
#Report bands that have high leave-one-out loading variance
imageData["grayBandRogueBands"] = [str(x) for x in rogueBands]
#Report histogram
imageData["grayBandHistogram"] = [[float(x) for x in bin_edges], [int(x) for x in hist]]
#emit the final statistics
storm.log("emiting Storm tuple")
storm.emit([localFileName, hdfsFileName, json.dumps(imageData)], stream="summaryStatistics")
storm.log("done with %s %s" % (localFileName, hdfsFileName))
mask = utilities.fixMask(mask,bands)
for bandKey, bandValue in bands.iteritems():
bands[bandKey] = bandValue[mask]
if 'HSI' in imageData["metadata"].keys():
wavelengths = {}
multipliers = {}
for w,wave in enumerate(imageData["metadata"][unicode("HSI")][unicode("wavelength")]):
wavelengths["B" + "%03d" % w] = float(wave)
multipliers["B" + "%03d" % w] = 1
else:
wavelengths = imageData["metadata"]["bandWavelength"]
multipliers = imageData["metadata"]["bandMultiplier"]
imageList = utilities.preprocessImage(bands, multipliers, wavelengths, imageData, selectBands=selectbands)
sys.stderr.write("This is the number of bands: %r\n" % len(imageList))
iDot,iPartial = main(imageList)
imageData["imageDot"] = iDot
imageData["imagePartial"] = iPartial
if noiseFlag.upper() == "TRUE":
noiseList = calcNoise(imageList,mask)
nDot,nPartial = main(noiseList)
imageData["noiseDot"] = nDot
imageData["noisePartial"] = nPartial
try:
regionKey = imageData["metadata"]["originalDirName"]
except KeyError:
regionKey = imageData["metadata"]["outputFile"]
