def process(self, tup):
localFileName = tup.values[0]
hdfsFileName = tup.values[1]
imageData = {}
imageData["metadata"] = None
storm.log("start processing %s %s" % (localFileName, hdfsFileName))
for key, sorter, interpretedValue in binaryhadoop.readFromHDFSiter(hdfsFileName):
if key == "metadata":
imageData["metadata"] = interpretedValue
bands = {}
storm.log(" read metadata")
elif key == "mask":
mask = utilities.rollMask(interpretedValue > 0)
numPixels = numpy.nonzero(mask)[0].size
storm.log(" read mask")
else:
bands[key] = interpretedValue[mask]
storm.log(" read band %s" % key)
if imageData["metadata"] is not None:
wavelengths = imageData["metadata"]["bandWavelength"]
multipliers = imageData["metadata"]["bandMultiplier"]
storm.log("making imageArray 1")
imageList = utilities.preprocessImage(bands, multipliers, wavelengths, imageData)
#find the covariance of the image bands
storm.log("making covariance")
imageCov = self.makeCovariance(imageList, numPixels)
#find the principal components (eigenvectors/values)
storm.log("making principle components 1")
imgV, imgP = numpy.linalg.eig(imageCov)
#
storm.log("making principle components 2")
indexList = numpy.argsort(-imgV)
imgV = imgV[indexList]
imgP = imgP[:,indexList]
storm.log("making variance percentage")
xVarianceComponents = 5
variancePercentage = [x/numpy.sum(imgV) for x in imgV][:xVarianceComponents]
storm.log("making rogue bands")
rogueBands = self.checkpca(imgP.T,xVarianceComponents)
storm.log("making gray bands")
bandGray = numpy.zeros(len(imageList[0]))
for band in imageList:
bandGray += (numpy.array(band))**2
bandGray = numpy.sort(bandGray)
bandPercent = 1
#The 99th percentile is removed to avoid skewing the mean
bandGray = bandGray[bandGray < numpy.percentile(bandGray,100-bandPercent)]
#Histogram is created
storm.log("making gray band histogram")
[hist,bin_edges] = numpy.histogram(bandGray,bins=100)
#Locate the peaks on the histogram
storm.log("making peaks and valleys")
peaks, valleys = self.findpeaks(hist,3,(bin_edges[:-1] + bin_edges[1:])/2)
#Find mean and standard deviation of all pixels
bandMean = numpy.mean(bandGray)
bandSigma = numpy.std(bandGray)
storm.log("making JSON output")
imageData["numPixels"] = int(numPixels)
imageData["grayBandMean"] = float(bandMean)
imageData["grayBandSigma"] = float(bandSigma)
#Report percentage of total pixels which lie beyond one standard deviation from mean
imageData["grayBandPlusOneSigma"] = float(numpy.sum(bandGray > (bandMean+bandSigma))/numpy.float(numPixels))
imageData["grayBandMinusOneSigma"] = float(numpy.sum(bandGray < (bandMean-bandSigma))/numpy.float(numPixels))
imageData["grayBandHistPeaks"] = [[float(x), int(y)] for x, y in peaks]
#imageData["grayBandHistValleys"] = [[float(x), int(y)] for x, y in valleys]
#PCA analysis and sum of first 5 principal components
imageData["grayBandExplainedVariance"] = [float(x) for x in variancePercentage]
#Report bands that have high leave-one-out loading variance
imageData["grayBandRogueBands"] = [str(x) for x in rogueBands]
#Report histogram
imageData["grayBandHistogram"] = [[float(x) for x in bin_edges], [int(x) for x in hist]]
#emit the final statistics
storm.log("emiting Storm tuple")
storm.emit([localFileName, hdfsFileName, json.dumps(imageData)], stream="summaryStatistics")
storm.log("done with %s %s" % (localFileName, hdfsFileName))
if not os.path.exists("/var/www/reports/{}-{}/gmm-knn/image_reports".format(year, month)):
os.mkdir("/var/www/reports/{}-{}/gmm-knn".format(year, month))
os.mkdir("/var/www/reports/{}-{}/gmm-knn/image_reports".format(year, month))
# makingReport = not os.path.exists("/var/www/reports/{}-{}/gmm-knn/image_reports/{}.html".format(year, month, shortName))
# makingImage = not os.path.exists("/var/www/reports/{}-{}/gmm-knn/image_reports/{}.svg".format(year, month, shortName))
makingReport = True
makingImage = True
print " " + ("" if makingReport else "NOT ") + "making report and " + ("" if makingImage else "NOT ") + "making image"; sys.stdout.flush()
if makingImage:
sawkeys = set()
for key, sorter, value in binaryhadoop.readFromHDFSiter(imageName):
sawkeys.add(key)
if key == "metadata":
metadata = value
elif key == "mask":
mask = value
numBands = len(metadata["bandNames"])
rgbDouble = numpy.empty([3] + list(mask.shape), dtype=numpy.double)
else:
if key == "B05":
index = 0
elif key == "B04":
index = 1
elif key == "B03":
index = 2
else:
