def testDifferentOutput(imgfile, imgfile2):
makeTestFile(imgfile2, withRat=False)
inRats = ratapplier.RatAssociations()
outRats = ratapplier.RatAssociations()
controls = ratapplier.RatApplierControls()
inRats.img = ratapplier.RatHandle(imgfile)
outRats.outimg = ratapplier.RatHandle(imgfile2)
controls.setBlockLength(3)
ratapplier.apply(myFuncDiffFile, inRats, outRats, controls=controls)
col = rat.readColumn(imgfile, 'Value')
colSqrd = rat.readColumn(imgfile2, 'sqrd')
ok = True
if (col**2 != colSqrd).any():
riostestutils.report(TESTNAME,
"sqrd incorrect, in differentFile output")
ok = False
return ok
def testReduceRat(imgfile, imgfile3):
"""
This test creates a new output image, with all odd pixel values
replaced with the even number above it. The RAT must then be copied across
with the same reduction performed. In this case, only the even numbered
rows are written
"""
# First we copy the raster, with the reduction of pixel values
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
infiles.inimg = imgfile
outfiles.outimg = imgfile3
# Make sure we use a format which actually supports RAT's
controls = applier.ApplierControls()
controls.setOutputDriverName('HFA')
applier.apply(rasterReduceFunc, infiles, outfiles, controls=controls)
# Now use ratapplier to reduce the RAT
inRats = ratapplier.RatAssociations()
outRats = ratapplier.RatAssociations()
controls = ratapplier.RatApplierControls()
inRats.img = ratapplier.RatHandle(imgfile)
outRats.outimg = ratapplier.RatHandle(imgfile3)
controls.setBlockLength(3)
ratapplier.apply(ratReduceFunc, inRats, outRats, controls=controls)
col = rat.readColumn(imgfile, 'Value')
colEven = col[::2]
colReduced = rat.readColumn(imgfile3, 'Value')[:len(colEven)]
ok = True
if (colEven != colReduced).any():
riostestutils.report(
TESTNAME, "Reduced RAT incorrect: %s, %s" % (colEven, colReduced))
ok = False
return ok
def testOutputSameFile(imgfile):
# Now test the ratapplier
inRats = ratapplier.RatAssociations()
outRats = ratapplier.RatAssociations()
controls = ratapplier.RatApplierControls()
inRats.img = ratapplier.RatHandle(imgfile)
outRats.img = inRats.img
controls.setBlockLength(5)
ratapplier.apply(myFunc, inRats, outRats, controls=controls)
col = rat.readColumn(imgfile, 'Value')
colSqrd = rat.readColumn(imgfile, 'sqrd')
ok = True
if (col**2 != colSqrd).any():
riostestutils.report(TESTNAME, "sqrd incorrect, in sameFile output")
ok = False
return ok
def testNewRat(imgfile4):
makeTestFile(imgfile4, withRat=False)
inRats = ratapplier.RatAssociations()
outRats = ratapplier.RatAssociations()
controls = ratapplier.RatApplierControls()
controls.setRowCount(256)
outRats.outimg = ratapplier.RatHandle(imgfile4)
controls.setBlockLength(3)
ratapplier.apply(myFuncNewRat, inRats, outRats, controls=controls)
col = rat.readColumn(imgfile4, 'newCol')
colIntended = numpy.arange(256, dtype=numpy.uint32)
ok = (col == colIntended).all()
if not ok:
riostestutils.report(TESTNAME,
"New RAT incorrect: %s, %s" % (col, colIntended))
return ok
def classifyWithinRATTiled(clumpsImg,
classesIntCol,
classesNameCol,
variables,
classifier=RandomForestClassifier(n_estimators=100,
max_features=3,
oob_score=True,
n_jobs=-1),
outColInt="OutClass",
outColStr="OutClassName",
roiCol=None,
roiVal=1,
classColours=None,
scaleVarsRange=False,
justFit=False):
"""
A function which will perform a classification within the RAT using a classifier from scikit-learn using the rios ratapplier interface allowing very large RATs to be processed.
:param clumpsImg: is the clumps image on which the classification is to be performed
:param classesIntCol: is the column with the training data as int values
:param classesNameCol: is the column with the training data as string class names
:param variables: is an array of column names which are to be used for the classification
:param classifier: is an instance of a scikit-learn classifier (e.g., RandomForests which is Default)
:param outColInt: is the output column name for the int class representation (Default: 'OutClass')
:param outColStr: is the output column name for the class names column (Default: 'OutClassName')
:param roiCol: is a column name for a column which specifies the region to be classified. If None ignored (Default: None)
:param roiVal: is a int value used within the roiCol to select a region to be classified (Default: 1)
:param classColours: is a python dict using the class name as the key along with arrays of length 3 specifying the RGB colours for the class.
:param scaleVarsRange: will rescale each variable independently to a range of 0-1 (default: False).
:param justFit: is a boolean specifying that the classifier should just be fitted to the data and not applied (Default: False; i.e., apply classification)
Example::
from sklearn.ensemble import ExtraTreesClassifier
from rsgislib.classification import classratutils
classifier = ExtraTreesClassifier(n_estimators=100, max_features=3, n_jobs=-1, verbose=0)
classColours = dict()
classColours['Forest'] = [0,138,0]
classColours['NonForest'] = [200,200,200]
variables = ['GreenAvg', 'RedAvg', 'NIR1Avg', 'NIR2Avg', 'NDVI']
classifyWithinRATTiled(clumpsImg, classesIntCol, classesNameCol, variables, classifier=classifier, classColours=classColours)
# With using range scaling.
classifyWithinRATTiled(clumpsImg, classesIntCol, classesNameCol, variables, classifier=classifier, classColours=classColours, scaleVarsRange=True)
"""
# Check gdal is available
if not haveGDALPy:
raise Exception(
"The GDAL python bindings required for this function could not be imported\n\t"
+ gdalErr)
# Check numpy is available
if not haveNumpy:
raise Exception(
"The numpy module is required for this function could not be imported\n\t"
+ numErr)
# Check rios rat is available
if not haveRIOSRat:
raise Exception(
"The RIOS rat tools are required for this function could not be imported\n\t"
+ riosRatErr)
# Check scikit-learn RF is available
if not haveSKLearnRF:
raise Exception(
"The scikit-learn random forests tools are required for this function could not be imported\n\t"
+ sklearnRFErr)
# Check scikit-learn pre-processing is available
if not haveSKLearnPreProcess:
raise Exception(
"The scikit-learn pre-processing tools are required for this function could not be imported\n\t"
+ sklearnPreProcessErr)
ratDataset = gdal.Open(clumpsImg, gdal.GA_Update)
# Read in training classes
classesInt = rat.readColumn(ratDataset, classesIntCol)
classesStr = rat.readColumn(ratDataset, classesNameCol)
ratDataset = None
validClassStr = classesStr[classesInt > 0]
validClassInt = classesInt[classesInt > 0]
#print(validClassInt.shape)
classNames = numpy.unique(validClassStr)
classes = numpy.zeros_like(classNames, dtype=numpy.int16)
i = 0
classNameIDs = dict()
for className in classNames:
classNameStr = str(className.decode())
if not classNameStr is '':
#print(validClassInt[validClassStr == className])
classes[i] = validClassInt[validClassStr == className][0]
classNameIDs[classNameStr] = classes[i]
#print("Class \'" + classNameStr + "\' has numerical " + str(classes[i]))
i = i + 1
trainLen = validClassInt.shape[0]
numVars = len(variables)
#print("Create numpy {} x {} array for training".format(trainLen, numVars))
trainData = numpy.zeros((trainLen, numVars), numpy.float64)
in_rats = ratapplier.RatAssociations()
out_rats = ratapplier.RatAssociations()
in_rats.inrat = ratapplier.RatHandle(clumpsImg)
otherargs = ratapplier.OtherArguments()
otherargs.vars = variables
otherargs.classIntCol = classesIntCol
otherargs.trainData = trainData
otherargs.trainDataOff = 0
print("Extract Training Data")
ratapplier.apply(_extractTrainDataFromRAT,
in_rats,
out_rats,
otherargs=otherargs,
controls=None)
print("100%")
validClassInt = validClassInt[numpy.isfinite(trainData).all(axis=1)]
validClassStr = validClassStr[numpy.isfinite(trainData).all(axis=1)]
trainData = trainData[numpy.isfinite(trainData).all(axis=1)]
print("Training data size: {} x {}".format(trainData.shape[0],
trainData.shape[1]))
print('Training Classifier')
classifier.fit(trainData, validClassInt)
print("Completed")
print('Calc Classifier Accuracy')
accVal = classifier.score(trainData, validClassInt)
print('Classifier Score = {}'.format(round(accVal * 100, 2)))
if not justFit:
print("Apply Classifier")
in_rats = ratapplier.RatAssociations()
out_rats = ratapplier.RatAssociations()
in_rats.inrat = ratapplier.RatHandle(clumpsImg)
out_rats.outrat = ratapplier.RatHandle(clumpsImg)
otherargs = ratapplier.OtherArguments()
otherargs.vars = variables
otherargs.classifier = classifier
otherargs.outColInt = outColInt
otherargs.outColStr = outColStr
otherargs.roiCol = roiCol
otherargs.roiVal = roiVal
otherargs.classColours = classColours
otherargs.classNameIDs = classNameIDs
ratapplier.apply(_applyClassifier,
in_rats,
out_rats,
otherargs=otherargs,
controls=None)
print("100%")
#!/usr/bin/env python
import argparse
from rios import ratapplier
import numpy
def classifyRAT(info, inputs, outputs):
outClass = numpy.zeros_like(inputs.inrat.b1Mean, dtype=numpy.int16)
# Check for loss of water bodies (1)
outClass = numpy.where(inputs.inrat.b1Mean < 1000, 1, outClass)
# Save output column to RAT
outputs.outrat.outClass = outClass
# Set up options
parser = argparse.ArgumentParser()
parser.add_argument("inclumps", nargs=1, type=str, help="Input clumps file")
args = parser.parse_args()
inRats = ratapplier.RatAssociations()
outRats = ratapplier.RatAssociations()
inRats.inrat = ratapplier.RatHandle(args.inclumps[0])
outRats.outrat = ratapplier.RatHandle(args.inclumps[0])
ratapplier.apply(classifyRAT, inRats, outRats)
def perform_analysis(self, scn_db_obj, sen_obj, plgin_objs):
logger.info("Processing Scene: {}".format(scn_db_obj.PID))
if scn_db_obj.Invalid:
return False, None, False
rsgis_utils = rsgislib.RSGISPyUtils()
eodd_utils = EODataDownUtils()
success = True
outputs = False
out_dict = None
if 'GenChngSummaryFeats' in plgin_objs:
if plgin_objs['GenChngSummaryFeats'].Completed and plgin_objs[
'GenChngSummaryFeats'].Outputs and plgin_objs[
'GenChngSummaryFeats'].Success:
scn_chng_info = plgin_objs['GenChngSummaryFeats'].ExtendedInfo
scn_unq_name = sen_obj.get_scn_unq_name_record(scn_db_obj)
out_vec_file = os.path.join(
self.params['outvecdir'],
"{}_chng_vec.gpkg".format(scn_unq_name))
if os.path.exists(out_vec_file):
delete_vector_file(out_vec_file)
if sen_obj.get_sensor_name() == 'LandsatGOOG':
scn_obs_date = scn_db_obj.Sensing_Time
elif sen_obj.get_sensor_name() == 'Sentinel2GOOG':
scn_obs_date = scn_db_obj.Sensing_Time
elif sen_obj.get_sensor_name() == 'Sentinel1ASF':
scn_obs_date = scn_db_obj.Acquisition_Date
else:
raise Exception("Did not recognise the sensor name...")
start_date = datetime.datetime(year=2019, month=4, day=30)
if scn_obs_date > start_date:
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
from rios import cuiprogress
progress_bar = cuiprogress.GDALProgressBar()
drv = gdal.GetDriverByName("GPKG")
if drv is None:
raise Exception("Driver GPKG is not avaiable.")
ds = drv.Create(out_vec_file, 0, 0, 0, gdal.GDT_Unknown)
if ds is None:
raise Exception(
"Could not create output file: {}.".format(
out_vec_file))
out_dict = dict()
for tile in scn_chng_info:
logger.debug("Processing tile {}...".format(tile))
clumps_img = scn_chng_info[tile]
in_rats = ratapplier.RatAssociations()
out_rats = ratapplier.RatAssociations()
in_rats.inrat = ratapplier.RatHandle(clumps_img)
lyr = ds.CreateLayer(tile, None, ogr.wkbPoint)
if lyr is None:
raise Exception(
"Could not create output layer: {}.".format(
tile))
field_uid_defn = ogr.FieldDefn("uid", ogr.OFTInteger)
if lyr.CreateField(field_uid_defn) != 0:
raise Exception("Could not create field: 'uid'.")
field_prop_chng_defn = ogr.FieldDefn(
"prop_chng", ogr.OFTReal)
if lyr.CreateField(field_prop_chng_defn) != 0:
raise Exception(
"Could not create field: 'prop_chng'.")
field_score_defn = ogr.FieldDefn(
"score", ogr.OFTInteger)
if lyr.CreateField(field_score_defn) != 0:
raise Exception("Could not create field: 'score'.")
# First Observation Date
field_firstobsday_defn = ogr.FieldDefn(
"firstobsday", ogr.OFTInteger)
if lyr.CreateField(field_firstobsday_defn) != 0:
raise Exception(
"Could not create field: 'firstobsday'.")
field_firstobsmonth_defn = ogr.FieldDefn(
"firstobsmonth", ogr.OFTInteger)
if lyr.CreateField(field_firstobsmonth_defn) != 0:
raise Exception(
"Could not create field: 'firstobsmonth'.")
field_firstobsyear_defn = ogr.FieldDefn(
"firstobsyear", ogr.OFTInteger)
if lyr.CreateField(field_firstobsyear_defn) != 0:
raise Exception(
"Could not create field: 'firstobsyear'.")
# Last Observation Date
field_lastobsday_defn = ogr.FieldDefn(
"lastobsday", ogr.OFTInteger)
if lyr.CreateField(field_lastobsday_defn) != 0:
raise Exception(
"Could not create field: 'lastobsday'.")
field_lastobsmonth_defn = ogr.FieldDefn(
"lastobsmonth", ogr.OFTInteger)
if lyr.CreateField(field_lastobsmonth_defn) != 0:
raise Exception(
"Could not create field: 'lastobsmonth'.")
field_lastobsyear_defn = ogr.FieldDefn(
"lastobsyear", ogr.OFTInteger)
if lyr.CreateField(field_lastobsyear_defn) != 0:
raise Exception(
"Could not create field: 'lastobsyear'.")
# Observation Date Where Score Reached 5
field_scr5obsday_defn = ogr.FieldDefn(
"scr5obsday", ogr.OFTInteger)
if lyr.CreateField(field_scr5obsday_defn) != 0:
raise Exception(
"Could not create field: 'scr5obsday'.")
field_scr5obsmonth_defn = ogr.FieldDefn(
"scr5obsmonth", ogr.OFTInteger)
if lyr.CreateField(field_scr5obsmonth_defn) != 0:
raise Exception(
"Could not create field: 'scr5obsmonth'.")
field_scr5obsyear_defn = ogr.FieldDefn(
"scr5obsyear", ogr.OFTInteger)
if lyr.CreateField(field_scr5obsyear_defn) != 0:
raise Exception(
"Could not create field: 'scr5obsyear'.")
lyr_defn = lyr.GetLayerDefn()
otherargs = ratapplier.OtherArguments()
otherargs.lyr = lyr
otherargs.lyr_defn = lyr_defn
ratcontrols = ratapplier.RatApplierControls()
ratcontrols.setProgress(progress_bar)
ratapplier.apply(_ratapplier_check_string_col_valid,
in_rats,
out_rats,
otherargs,
controls=ratcontrols)
# Update (create) the JSON LUT file.
lut_file_name = "gmw_{}_lut.json".format(tile)
lut_file_path = os.path.join(self.params["outlutdir"],
lut_file_name)
eodd_utils.get_file_lock(lut_file_path,
sleep_period=1,
wait_iters=120,
use_except=True)
if os.path.exists(lut_file_path):
lut_dict = rsgis_utils.readJSON2Dict(lut_file_path)
else:
lut_dict = dict()
obs_date_iso_str = scn_obs_date.isoformat()
lut_dict[obs_date_iso_str] = dict()
lut_dict[obs_date_iso_str]["file"] = out_vec_file
lut_dict[obs_date_iso_str]["layer"] = tile
rsgis_utils.writeDict2JSON(lut_dict, lut_file_path)
eodd_utils.release_file_lock(lut_file_path)
out_dict[tile] = out_vec_file
ds = None
outputs = True
success = True
else:
logger.debug(
"Scene is within window used to mask change outside of range."
)
else:
logger.debug(
"No change features available as outputs from previous steps..."
)
else:
logger.debug(
"GenChngSummaryFeats was not available so previous step had not run..."
)
return success, out_dict, outputs