def step(taskName, parameterByName, folderStore, options):
# Get
windowGeoLength = parameterByName['window length in meters']
windowLabel = parameterByName['window label']
windowCenterPath = parameterByName['window center path']
imageName = parameterByName['image name']
imagePath = folderStore.getImagePath(imageName)
imageInformation = folderStore.getImageInformation(imageName)
# Prepare
geoCenters, spatialReference = point_store.load(windowCenterPath)
multispectralImage = imageInformation.getMultispectralImage()
panchromaticImage = imageInformation.getPanchromaticImage()
# Set
targetWindowPath = folderStore.fillWindowPath(taskName)
exampleInformation = {}
# Extract
if not options.is_test:
dataset = window_process.extract(targetWindowPath, geoCenters, windowLabel, windowGeoLength, multispectralImage, panchromaticImage)
exampleInformation['count'] = dataset.countSamples()
# Save
store.saveInformation(targetWindowPath, {
'windows': {
'window length in meters': windowGeoLength,
'spatial reference': spatialReference,
},
'parameters': {
'window label': windowLabel,
'window center path': windowCenterPath,
'image name': imageName,
'image path': imagePath,
},
'examples': exampleInformation,
})
def step(taskName, parameterByName, folderStore, options):
# Get parameters
classifierName = parameterByName['classifier name']
classifierPath = folderStore.getClassifierPath(classifierName)
classifierInformation = folderStore.getClassifierInformation(classifierName)
windowLengthInMeters = classifierInformation.getWindowLengthInMeters()
scanRatio = parameterByName['scan ratio']
regionName = parameterByName['region name']
regionPath = folderStore.getRegionPath(regionName)
regionInformation = folderStore.getRegionInformation(regionName)
regionDataset = regionInformation.getRegionDataset()
# Prepare
imageName = regionInformation.getImageName()
imagePath = folderStore.getImagePath(imageName)
imageInformation = folderStore.getImageInformation(imageName)
multispectralImagePath = imageInformation.getMultispectralImagePath()
multispectralImage = image_store.load(multispectralImagePath)
panchromaticImagePath = imageInformation.getPanchromaticImagePath()
positiveLocationPath = imageInformation.getPositiveLocationPath()
regionFrames = regionDataset.getRegionFrames()
# Record
targetProbabilityPath = folderStore.fillProbabilityPath(taskName)
probabilityInformation = {
'classifier': {
'name': classifierName,
'path': classifierPath,
},
'parameters': {
'window length in meters': windowLengthInMeters,
'scan ratio': scanRatio,
},
'probability': {
'region name': regionName,
'region path': regionPath,
'image name': imageName,
'image path': imagePath,
},
}
store.saveInformation(targetProbabilityPath, probabilityInformation)
# If this is not a test,
if not options.is_test:
# Scan
info = classifier.scan(targetProbabilityPath, classifierPath, multispectralImagePath, panchromaticImagePath, scanRatio, regionFrames)
# Save
print 'Saving probability matrix as a shapefile...'
probability_store.saveShapefile(targetProbabilityPath, targetProbabilityPath, image_store.load(multispectralImagePath), windowLengthInMeters)
# Evaluate
print 'Evaluating windows...'
windowPerformance, wrongPixelCenters, actualPixelPointMachine = evaluation_process.evaluateWindows(targetProbabilityPath, positiveLocationPath, multispectralImagePath, windowLengthInMeters)
probabilityInformation['performance'] = windowPerformance
probabilityInformation['performance'].update(info)
else:
wrongPixelCenters = []
windowPixelWidth, windowPixelHeight = multispectralImage.convertGeoDimensionsToPixelDimensions(windowLengthInMeters, windowLengthInMeters)
actualPixelPointMachine = point_process.PointMachine([], 'INTEGER', windowPixelWidth, windowPixelHeight)
# Update
store.saveInformation(targetProbabilityPath, probabilityInformation)
# Build patch
patch_process.buildPatchFromScan(wrongPixelCenters, folderStore, taskName, targetProbabilityPath, multispectralImagePath, panchromaticImagePath, actualPixelPointMachine, classifierInformation, options.is_test)
def step(taskName, parameterByName, folderStore, options):
# Get parameters
trainingSize = parameterByName.get("training size")
testSize = parameterByName.get("test size")
positiveFraction = parameterByName.get("positive fraction")
# Get names
windowNames = parameterByName.get("window names", [])
windowPaths = map(folderStore.getWindowPath, windowNames)
windowFolderPaths = map(os.path.dirname, windowPaths)
windowInformations = map(folderStore.getWindowInformation, windowNames)
patchNames = parameterByName.get("patch names", [])
patchPaths = map(folderStore.getPatchPath, patchNames)
patchFolderPaths = map(os.path.dirname, patchPaths)
patchInformations = map(folderStore.getPatchInformation, patchNames)
# Set
sourceInformations = windowInformations + patchInformations
sourceFolderPaths = windowFolderPaths + patchFolderPaths
# Make sure that each dataset has the same windowGeoLength
windowLengthInMeters = store.validateSame(
[x.getWindowLengthInMeters() for x in sourceInformations],
"Datasets must have the same window length in meters: %s" % taskName,
)
# Make sure that each dataset has the same spatialReference
spatialReference = store.validateSame(
[x.getSpatialReference() for x in sourceInformations],
"Datasets must have the same spatial reference: %s" % taskName,
)
# Set
targetDatasetPath = folderStore.fillDatasetPath(taskName)
targetDatasetFolderPath = os.path.dirname(targetDatasetPath)
# Record
information = {
"parameters": {"training size": trainingSize, "test size": testSize, "positive fraction": positiveFraction},
"windows": {"spatial reference": spatialReference, "window length in meters": windowLengthInMeters},
"sources": {
"window names": store.stringifyList(windowNames),
"window paths": store.stringifyList(windowPaths),
"patch names": store.stringifyList(patchNames),
"patch paths": store.stringifyList(patchPaths),
},
}
# Combine training and test sets
if not options.is_test:
print "Combining datasets...\n\ttargetDatasetPath = %s" % targetDatasetPath
information["training set"] = sample_process.combineDatasets(
sample_process.makeTrainingPath(targetDatasetFolderPath),
map(sample_process.makeTrainingPath, sourceFolderPaths),
trainingSize,
positiveFraction,
).getStatistics()
information["test set"] = sample_process.combineDatasets(
sample_process.makeTestPath(targetDatasetFolderPath),
map(sample_process.makeTestPath, sourceFolderPaths),
testSize,
positiveFraction,
).getStatistics()
# Save
store.saveInformation(targetDatasetPath, information)
def step(taskName, parameterByName, folderStore, options):
# Get probabilityInformation
probabilityName = parameterByName['probability name']
probabilityPath = folderStore.getProbabilityPath(probabilityName)
probabilityInformation = probability_store.Information(probabilityPath)
# Get imageInformation
imageName = probabilityInformation.getImageName()
imageInformation = folderStore.getImageInformation(imageName)
# Get parameters
positiveLocationPath = imageInformation.getPositiveLocationPath()
multispectralImage = imageInformation.getMultispectralImage()
spatialReference = multispectralImage.getSpatialReference()
evaluationRadiusInMeters = parameterByName['evaluation radius in meters']
evaluationDiameterInMeters = evaluationRadiusInMeters * 2
iterationCountPerBurst = parameterByName['iteration count per burst']
maximumDiameterInMeters = parameterByName['maximum diameter in meters']
minimumDiameterInMeters = parameterByName['minimum diameter in meters']
# Run
targetLocationPath = folderStore.fillLocationPath(taskName)
targetFolderPath = os.path.dirname(targetLocationPath)
locationInformation = {
'location': {
'path': targetLocationPath,
'spatial reference': spatialReference,
},
'probability': {
'name': probabilityName,
'path': probabilityPath,
},
'parameters': {
'iteration count per burst': iterationCountPerBurst,
'maximum diameter in meters': maximumDiameterInMeters,
'minimum diameter in meters': minimumDiameterInMeters,
'evaluation radius in meters': evaluationRadiusInMeters,
},
}
if not options.is_test:
# Cluster
print 'Clustering locations...'
probability_process.cluster(targetLocationPath, probabilityPath, iterationCountPerBurst, maximumDiameterInMeters, minimumDiameterInMeters)
# If scanning has finished,
if probabilityInformation.hasPerformance():
print 'Evaluating locations...'
# Load regions
regionPath = probabilityInformation.getRegionPath()
regionGeoFrames = region_store.loadShapefile(regionPath, multispectralImage, withGeoToPixelConversion=False)
# Evaluate locations
locationInformation['performance'] = evaluation_process.evaluateLocations(targetFolderPath, evaluationDiameterInMeters, positiveLocationPath, targetLocationPath, regionGeoFrames)
# Record
store.saveInformation(targetLocationPath, locationInformation)
def step(taskName, parameterByName, folderStore, options):
# Get parameters
datasetName = parameterByName['dataset name']
datasetPath = folderStore.getDatasetPath(datasetName)
datasetFolderPath = os.path.dirname(datasetPath)
# Set
trainingPath = sample_process.makeTrainingPath(datasetFolderPath)
testPath = sample_process.makeTestPath(datasetFolderPath)
datasetInformation = dataset_store.Information(trainingPath)
if not options.is_test:
# Make sure that training and test sets are not empty
if not datasetInformation.getTrainingCount():
raise script_process.ScriptError('Empty training set: %s' % trainingPath)
if not datasetInformation.getTestCount():
raise script_process.ScriptError('Empty test set: %s' % testPath)
# Pack
classifierModule = store.getLibraryModule(parameterByName['classifier module name'])
featureModule = store.getLibraryModule(parameterByName['feature module name'])
featureClass = getattr(featureModule, parameterByName['feature class name'])
# Run
targetClassifierPath = folderStore.fillClassifierPath(taskName)
if not isTest:
# Build classifier
resultByName = classifier.build(targetClassifierPath, classifierModule, featureClass(), trainingPath, testPath, parameterByName)
else:
resultByName = {}
# Record
makeDictionary = lambda keys: dict((key, parameterByName[key]) for key in keys if key in parameterByName)
parameterInformation = makeDictionary(['classifier module name', 'feature module name', 'feature class name'])
parameterInformation.update(makeDictionary(classifierModule.relevantParameterNames))
store.saveInformation(targetClassifierPath, {
'parameters': parameterInformation,
'dataset': {
'name': datasetName,
'path': datasetPath,
},
'windows': {
'training': trainingPath,
'test': testPath,
'window length in meters': datasetInformation.getWindowLengthInMeters(),
'spatial reference': datasetInformation.getSpatialReference(),
},
'performance': resultByName,
})
def defineRegions(targetRegionPath, multispectralImagePath, panchromaticImagePath, parameterByName, options=None):
# Load
targetTestRegionPath = targetRegionPath + '-test'
multispectralImage = image_store.load(str(multispectralImagePath))
panchromaticImage = image_store.load(str(panchromaticImagePath))
regionPath = parameterByName.get('region path')
multispectralRegionFrames = parameterByName.get('multispectral region frames')
windowLengthInMeters = parameterByName.get('window length in meters')
regionLengthInWindows = parameterByName.get('region length in windows')
testFractionPerRegion = parameterByName['test fraction per region']
coverageFraction = parameterByName.get('coverage fraction', 1)
coverageFrequency = int(1 / coverageFraction)
coverageOffset = parameterByName.get('coverage offset', 0)
# If regionPath is defined, use it
if regionPath:
regionGenerator = (x for x in region_store.loadShapefile(regionPath, multispectralImage))
# If multispectralRegionFrames are defined, use them
elif multispectralRegionFrames:
regionGenerator = (x for x in multispectralRegionFrames)
# Otherwise, prepare regionGenerator
else:
regionGenerator = region_store.makeRegionGenerator(multispectralImage, panchromaticImage, regionLengthInWindows, windowLengthInMeters)
# Save regions
regionDataset = region_store.create(targetRegionPath)
testRegionDataset = region_store.create(targetTestRegionPath)
if options and not options.is_test:
for regionIndex, regionWindow in itertools.izip(itertools.count(1), regionGenerator):
if (regionIndex + coverageOffset) % coverageFrequency == 0:
regionDataset.addRegion(regionWindow)
regionFrame = region_store.getMultispectralPixelFrame(regionWindow)
testRegionDataset.addFrame(region_process.placeTestRegion(regionFrame, testFractionPerRegion))
# Save as shapefiles
regionDataset.saveShapefile(targetRegionPath, multispectralImage)
testRegionDataset.saveShapefile(targetTestRegionPath, multispectralImage)
# Prepare information
information = {
'parameters': {
'multispectral image path': multispectralImagePath,
'panchromatic image path': panchromaticImagePath,
'test fraction per region': testFractionPerRegion,
'window length in meters': windowLengthInMeters,
},
'regions': {
'path': regionDataset.getPath(),
'count': regionDataset.count(),
},
'test regions': {
'path': testRegionDataset.getPath(),
'count': testRegionDataset.count(),
},
}
if regionPath:
information['parameters'].update({
'region path': regionPath,
})
elif multispectralRegionFrames:
information['parameters'].update({
'multispectral region frames': store.stringifyNestedList(multispectralRegionFrames),
})
else:
information['parameters'].update({
'region length in windows': regionLengthInWindows,
'coverage fraction': coverageFraction,
'coverage offset': coverageOffset,
})
# Save information
store.saveInformation(targetRegionPath, information)
def step(taskName, parameterByName, folderStore, options):
# Get parameters
imageName = parameterByName['image name']
imagePath = folderStore.getImagePath(imageName)
imageInformation = folderStore.getImageInformation(imageName)
multispectralImagePath = imageInformation.getMultispectralImagePath()
multispectralImage = image_store.load(multispectralImagePath)
scanRatio = float(parameterByName['scan ratio'])
classifierName = parameterByName['classifier name']
classifierPath = folderStore.getClassifierPath(classifierName)
classifierInformation = folderStore.getClassifierInformation(classifierName)
windowLengthInMeters = classifierInformation.getWindowLengthInMeters()
panchromaticImagePath = imageInformation.getPanchromaticImagePath()
positiveLocationPath = imageInformation.getPositiveLocationPath()
coverageFraction = parameterByName.get('coverage fraction', 1)
# Record
targetProbabilityPath = folderStore.fillProbabilityPath(taskName)
regionPath = targetProbabilityPath + '_region'
probabilityInformation = {
'classifier': {
'name': classifierName,
'path': classifierPath,
},
'parameters': {
'window length in meters': windowLengthInMeters,
'scan ratio': scanRatio,
'coverage fraction': coverageFraction,
},
'probability': {
'region path': regionPath,
'image name': imageName,
'image path': imagePath,
},
}
# Run
store.saveInformation(targetProbabilityPath, probabilityInformation)
if not options.is_test:
# Frame
xMax = multispectralImage.width
yMax = multispectralImage.height
xMargin = int(xMax * (1 - coverageFraction) / 2)
yMargin = int(yMax * (1 - coverageFraction) / 2)
regionFrames = [(xMargin, yMargin, xMax - xMargin, yMax - yMargin)]
regionDataset = region_store.save(regionPath, regionFrames)
regionDataset.saveShapefile(regionPath, multispectralImage)
# Scan
info = classifier.scan(targetProbabilityPath, classifierPath, multispectralImagePath, panchromaticImagePath, scanRatio, regionFrames)
# Save
print 'Saving probability matrix as a shapefile...'
probability_store.saveShapefile(targetProbabilityPath, targetProbabilityPath, multispectralImage, windowLengthInMeters)
# Evaluate windows
windowPerformance, wrongPixelCenters, actualPixelPointMachine = evaluation_process.evaluateWindows(targetProbabilityPath, positiveLocationPath, multispectralImagePath, windowLengthInMeters)
probabilityInformation['performance'] = windowPerformance
probabilityInformation['performance'].update(info)
else:
wrongPixelCenters = []
windowPixelWidth, windowPixelHeight = multispectralImage.convertGeoDimensionsToPixelDimensions(windowLengthInMeters, windowLengthInMeters)
actualPixelPointMachine = point_process.PointMachine([], 'INTEGER', windowPixelWidth, windowPixelHeight)
# Update
store.saveInformation(targetProbabilityPath, probabilityInformation)
# Save patch
patch_process.buildPatchFromScan(wrongPixelCenters, folderStore, taskName, targetProbabilityPath, multispectralImagePath, panchromaticImagePath, actualPixelPointMachine, classifierInformation, options.is_test)
def step(taskName, parameterByName, folderStore, options):
# Get
patchCountPerRegion = parameterByName['patch count per region']
minimumPercentCorrect = parameterByName['minimum percent correct']
probabilityName = parameterByName['probability name']
probabilityPath = folderStore.getProbabilityPath(probabilityName)
probabilityInformation = probability_store.Information(probabilityPath)
imageName = probabilityInformation.getImageName()
imageInformation = folderStore.getImageInformation(imageName)
multispectralImage = imageInformation.getMultispectralImage()
panchromaticImage = imageInformation.getPanchromaticImage()
positiveLocationPath = imageInformation.getPositiveLocationPath()
positiveGeoLocations, spatialReference = point_store.load(positiveLocationPath)
# If the probability is from scanning the entire image,
if not probabilityInformation.hasRegionName():
regionPath = folderStore.getRegionPath(parameterByName['region name'])
# Get regionDataset
else:
regionPath = probabilityInformation.getRegionPath()
regionInformation = region_store.Information(regionPath)
regionFrames = regionInformation.getRegionDataset().getRegionFrames()
regionCount = len(regionFrames)
testRegionDataset = regionInformation.getTestRegionDataset()
# Get windowPixelDimensions
windowLengthInMeters = probabilityInformation.getWindowLengthInMeters()
windowPixelDimensions = multispectralImage.convertGeoDimensionsToPixelDimensions(windowLengthInMeters, windowLengthInMeters)
# Get classifierInformation
classifierPath = probabilityInformation.getClassifierPath()
classifierInformation = classifier.Information(classifierPath)
# Record
information = {
'patches': {
'patch count per region': patchCountPerRegion,
'minimum percent correct': minimumPercentCorrect,
'probability name': probabilityName,
'probability path': probabilityPath,
},
'windows': {
'window length in meters': windowLengthInMeters,
'spatial reference': spatialReference,
},
}
# Set
targetPatchPath = folderStore.fillPatchPath(taskName)
targetPatchFolderPath = os.path.dirname(targetPatchPath)
if not options.is_test:
# Make pixelPointMachines
trainingPixelPointMachine = point_process.makePixelPointMachine(classifierInformation.getTrainingDataset().getGeoCenters(), windowLengthInMeters, multispectralImage)
testPixelPointMachine = point_process.makePixelPointMachine(classifierInformation.getTestDataset().getGeoCenters(), windowLengthInMeters, multispectralImage)
actualPixelPointMachine = point_process.makePixelPointMachine(positiveGeoLocations, windowLengthInMeters, multispectralImage)
# Get badRegionFrames
print 'Finding regions with poor performance...'
isGoodWindow = actualPixelPointMachine.getPointsInsideWindow
probabilityPacks = probability_store.load(probabilityPath)
performances = analyzeRegions(regionFrames, probabilityPacks, isGoodWindow)
badRegionFrames = [x[0] for x in itertools.izip(regionFrames, performances) if x[1] < minimumPercentCorrect]
badRegionCount = len(badRegionFrames)
# Save badRegionFrames
badRegionDataset = region_store.save(targetPatchPath, badRegionFrames)
badRegionDataset.saveShapefile(targetPatchPath, multispectralImage)
# Make patch window centers
print 'Sampling from regions with poor performance...'
testPatchPixelCenters = []
trainingPatchPixelCenters = []
for regionFrame in badRegionFrames:
# Load badCenters
testRegionFrame = testRegionDataset.getFrameInsideFrame(regionFrame)
regionalTrainingPixelCenters = trainingPixelPointMachine.getPointsInsideFrame(regionFrame)
regionalTestPixelCenters = testPixelPointMachine.getPointsInsideFrame(testRegionFrame)
badCenters = regionalTrainingPixelCenters + regionalTestPixelCenters
# Generate patch window centers
centerMachine = sample_process.CenterMachine(regionFrame, windowPixelDimensions, badCenters)
for patchWindowPixelCenter in centerMachine.makeCenters(patchCountPerRegion):
if point_process.isInsideRegion(patchWindowPixelCenter, testRegionFrame):
testPatchPixelCenters.append(patchWindowPixelCenter)
else:
trainingPatchPixelCenters.append(patchWindowPixelCenter)
# Define
buildPatch = lambda makePath, pixelCenters: patch_process.buildPatch(makePath(targetPatchFolderPath), pixelCenters, actualPixelPointMachine, multispectralImage, panchromaticImage, windowLengthInMeters)
# Produce training and test sets
information['training set'] = buildPatch(sample_process.makeTrainingPath, trainingPatchPixelCenters).getStatistics()
information['test set'] = buildPatch(sample_process.makeTestPath, testPatchPixelCenters).getStatistics()
# Record
information['performance'] = {
'bad region count': badRegionCount,
'region count': regionCount,
'percent correct': (regionCount - badRegionCount) / float(regionCount),
}
information['performance by region'] = dict(('%s %s %s %s' % x[0], x[1]) for x in itertools.izip(regionFrames, performances))
# Save
store.saveInformation(targetPatchPath, information)
def sampleWindows(targetWindowPath, region, location, parameterByName, options=None):
# Get parameters
exampleCountPerRegion = parameterByName['example count per region']
multispectralPixelShiftValue = parameterByName['multispectral pixel shift value']
shiftCount = parameterByName['shift count']
# Prepare regionFrames
regionSet = region.getDataset()
# regionDataset = region_store.load(region.path)
regionFrames = regionDataset.getRegionFrames()
regionFrameCount = len(regionFrames)
# Prepare counts
testRegionSet = region.getTestDataset()
# testRegionDataset = region_store.load(regionInformation.getTestRegionPath())
testFractionPerRegion = regionInformation.getTestFractionPerRegion()
# Load imageDataset
imagePath = folderStore.getImagePath(regionInformation.getImageName())
imageInformation = image_store.Information(imagePath)
multispectralImage = image_store.load(imageInformation.getMultispectralImagePath())
panchromaticImage = image_store.load(imageInformation.getPanchromaticImagePath())
# Load locations
positiveGeoLocations, spatialReference = point_store.load(imageInformation.getPositiveLocationPath())
# Convert
windowLengthInMeters = regionInformation.getWindowLengthInMeters()
windowPixelDimensions = multispectralImage.convertGeoDimensionsToPixelDimensions(windowLengthInMeters, windowLengthInMeters)
positivePixels = multispectralImage.convertGeoLocationsToPixelLocations(positiveGeoLocations)
# Place examples
exampleMachine = region_process.ExampleMachine(positivePixels, exampleCountPerRegion, testFractionPerRegion, testRegionDataset, windowPixelDimensions, multispectralPixelShiftValue, shiftCount)
examplePacks = []
if options and not options.is_test:
print 'Placing examples in %s regions for "%s"...' % (regionFrameCount, taskName)
for regionFrame in regionFrames:
examplePacks.append(exampleMachine.placeInFrame(regionFrame))
exampleCount = len(examplePacks)
if exampleCount % 10 == 0:
view.printPercentUpdate(exampleCount, regionFrameCount)
view.printPercentFinal(regionFrameCount)
exampleInformation = {}
trainingPaths = []
testPaths = []
# Set
targetWindowFolderPath = os.path.dirname(targetWindowPath)
if options and not options.is_test:
# For each exampleName,
for exampleName in examplePacks[0].keys():
# Convert
examplePixelLocations = sum((x[exampleName] for x in examplePacks), [])
exampleGeoLocations = multispectralImage.convertPixelLocationsToGeoLocations(examplePixelLocations)
examplePath = os.path.join(targetWindowFolderPath, exampleName)
exampleLabel = 1 if 'positive' in exampleName else 0
# Save
point_store.save(examplePath, exampleGeoLocations, spatialReference)
exampleInformation[exampleName + ' count'] = len(examplePixelLocations)
# Extract
print 'Extracting %s windows for %s...' % (len(examplePixelLocations), exampleName)
window_process.extract(examplePath, exampleGeoLocations, exampleLabel, windowLengthInMeters, multispectralImage, panchromaticImage)
(testPaths if 'test' in exampleName else trainingPaths).append(examplePath)
# Record
information = {
'windows': {
'window length in meters': windowLengthInMeters,
'spatial reference': spatialReference,
},
'regions': {
'name': regionName,
'path': regionPath,
'count': regionFrameCount,
},
'examples': exampleInformation,
}
# Combine
if options and not options.is_test:
information['training set'] = sample_process.combineDatasets(sample_process.makeTrainingPath(targetWindowFolderPath), trainingPaths).getStatistics()
information['test set'] = sample_process.combineDatasets(sample_process.makeTestPath(targetWindowFolderPath), testPaths).getStatistics()
# Save information
store.saveInformation(targetWindowPath, information)
