def step(taskName, parameterByName, folderStore, options):
# Get parameters
trainingSize = parameterByName.get("training size")
testSize = parameterByName.get("test size")
positiveFraction = parameterByName.get("positive fraction")
# Get names
windowNames = parameterByName.get("window names", [])
windowPaths = map(folderStore.getWindowPath, windowNames)
windowFolderPaths = map(os.path.dirname, windowPaths)
windowInformations = map(folderStore.getWindowInformation, windowNames)
patchNames = parameterByName.get("patch names", [])
patchPaths = map(folderStore.getPatchPath, patchNames)
patchFolderPaths = map(os.path.dirname, patchPaths)
patchInformations = map(folderStore.getPatchInformation, patchNames)
# Set
sourceInformations = windowInformations + patchInformations
sourceFolderPaths = windowFolderPaths + patchFolderPaths
# Make sure that each dataset has the same windowGeoLength
windowLengthInMeters = store.validateSame(
[x.getWindowLengthInMeters() for x in sourceInformations],
"Datasets must have the same window length in meters: %s" % taskName,
)
# Make sure that each dataset has the same spatialReference
spatialReference = store.validateSame(
[x.getSpatialReference() for x in sourceInformations],
"Datasets must have the same spatial reference: %s" % taskName,
)
# Set
targetDatasetPath = folderStore.fillDatasetPath(taskName)
targetDatasetFolderPath = os.path.dirname(targetDatasetPath)
# Record
information = {
"parameters": {"training size": trainingSize, "test size": testSize, "positive fraction": positiveFraction},
"windows": {"spatial reference": spatialReference, "window length in meters": windowLengthInMeters},
"sources": {
"window names": store.stringifyList(windowNames),
"window paths": store.stringifyList(windowPaths),
"patch names": store.stringifyList(patchNames),
"patch paths": store.stringifyList(patchPaths),
},
}
# Combine training and test sets
if not options.is_test:
print "Combining datasets...\n\ttargetDatasetPath = %s" % targetDatasetPath
information["training set"] = sample_process.combineDatasets(
sample_process.makeTrainingPath(targetDatasetFolderPath),
map(sample_process.makeTrainingPath, sourceFolderPaths),
trainingSize,
positiveFraction,
).getStatistics()
information["test set"] = sample_process.combineDatasets(
sample_process.makeTestPath(targetDatasetFolderPath),
map(sample_process.makeTestPath, sourceFolderPaths),
testSize,
positiveFraction,
).getStatistics()
# Save
store.saveInformation(targetDatasetPath, information)
def step(taskName, parameterByName, folderStore, options):
# Get parameters
datasetName = parameterByName['dataset name']
datasetPath = folderStore.getDatasetPath(datasetName)
datasetFolderPath = os.path.dirname(datasetPath)
# Set
trainingPath = sample_process.makeTrainingPath(datasetFolderPath)
testPath = sample_process.makeTestPath(datasetFolderPath)
datasetInformation = dataset_store.Information(trainingPath)
if not options.is_test:
# Make sure that training and test sets are not empty
if not datasetInformation.getTrainingCount():
raise script_process.ScriptError('Empty training set: %s' % trainingPath)
if not datasetInformation.getTestCount():
raise script_process.ScriptError('Empty test set: %s' % testPath)
# Pack
classifierModule = store.getLibraryModule(parameterByName['classifier module name'])
featureModule = store.getLibraryModule(parameterByName['feature module name'])
featureClass = getattr(featureModule, parameterByName['feature class name'])
# Run
targetClassifierPath = folderStore.fillClassifierPath(taskName)
if not isTest:
# Build classifier
resultByName = classifier.build(targetClassifierPath, classifierModule, featureClass(), trainingPath, testPath, parameterByName)
else:
resultByName = {}
# Record
makeDictionary = lambda keys: dict((key, parameterByName[key]) for key in keys if key in parameterByName)
parameterInformation = makeDictionary(['classifier module name', 'feature module name', 'feature class name'])
parameterInformation.update(makeDictionary(classifierModule.relevantParameterNames))
store.saveInformation(targetClassifierPath, {
'parameters': parameterInformation,
'dataset': {
'name': datasetName,
'path': datasetPath,
},
'windows': {
'training': trainingPath,
'test': testPath,
'window length in meters': datasetInformation.getWindowLengthInMeters(),
'spatial reference': datasetInformation.getSpatialReference(),
},
'performance': resultByName,
})
def sampleWindows(targetWindowPath, region, location, parameterByName, options=None):
# Get parameters
exampleCountPerRegion = parameterByName['example count per region']
multispectralPixelShiftValue = parameterByName['multispectral pixel shift value']
shiftCount = parameterByName['shift count']
# Prepare regionFrames
regionSet = region.getDataset()
# regionDataset = region_store.load(region.path)
regionFrames = regionDataset.getRegionFrames()
regionFrameCount = len(regionFrames)
# Prepare counts
testRegionSet = region.getTestDataset()
# testRegionDataset = region_store.load(regionInformation.getTestRegionPath())
testFractionPerRegion = regionInformation.getTestFractionPerRegion()
# Load imageDataset
imagePath = folderStore.getImagePath(regionInformation.getImageName())
imageInformation = image_store.Information(imagePath)
multispectralImage = image_store.load(imageInformation.getMultispectralImagePath())
panchromaticImage = image_store.load(imageInformation.getPanchromaticImagePath())
# Load locations
positiveGeoLocations, spatialReference = point_store.load(imageInformation.getPositiveLocationPath())
# Convert
windowLengthInMeters = regionInformation.getWindowLengthInMeters()
windowPixelDimensions = multispectralImage.convertGeoDimensionsToPixelDimensions(windowLengthInMeters, windowLengthInMeters)
positivePixels = multispectralImage.convertGeoLocationsToPixelLocations(positiveGeoLocations)
# Place examples
exampleMachine = region_process.ExampleMachine(positivePixels, exampleCountPerRegion, testFractionPerRegion, testRegionDataset, windowPixelDimensions, multispectralPixelShiftValue, shiftCount)
examplePacks = []
if options and not options.is_test:
print 'Placing examples in %s regions for "%s"...' % (regionFrameCount, taskName)
for regionFrame in regionFrames:
examplePacks.append(exampleMachine.placeInFrame(regionFrame))
exampleCount = len(examplePacks)
if exampleCount % 10 == 0:
view.printPercentUpdate(exampleCount, regionFrameCount)
view.printPercentFinal(regionFrameCount)
exampleInformation = {}
trainingPaths = []
testPaths = []
# Set
targetWindowFolderPath = os.path.dirname(targetWindowPath)
if options and not options.is_test:
# For each exampleName,
for exampleName in examplePacks[0].keys():
# Convert
examplePixelLocations = sum((x[exampleName] for x in examplePacks), [])
exampleGeoLocations = multispectralImage.convertPixelLocationsToGeoLocations(examplePixelLocations)
examplePath = os.path.join(targetWindowFolderPath, exampleName)
exampleLabel = 1 if 'positive' in exampleName else 0
# Save
point_store.save(examplePath, exampleGeoLocations, spatialReference)
exampleInformation[exampleName + ' count'] = len(examplePixelLocations)
# Extract
print 'Extracting %s windows for %s...' % (len(examplePixelLocations), exampleName)
window_process.extract(examplePath, exampleGeoLocations, exampleLabel, windowLengthInMeters, multispectralImage, panchromaticImage)
(testPaths if 'test' in exampleName else trainingPaths).append(examplePath)
# Record
information = {
'windows': {
'window length in meters': windowLengthInMeters,
'spatial reference': spatialReference,
},
'regions': {
'name': regionName,
'path': regionPath,
'count': regionFrameCount,
},
'examples': exampleInformation,
}
# Combine
if options and not options.is_test:
information['training set'] = sample_process.combineDatasets(sample_process.makeTrainingPath(targetWindowFolderPath), trainingPaths).getStatistics()
information['test set'] = sample_process.combineDatasets(sample_process.makeTestPath(targetWindowFolderPath), testPaths).getStatistics()
# Save information
store.saveInformation(targetWindowPath, information)