コード例 #1
0
def testOutputSameFile(imgfile):
    # Now test the ratapplier
    inRats = ratapplier.RatAssociations()
    outRats = ratapplier.RatAssociations()
    controls = ratapplier.RatApplierControls()

    inRats.img = ratapplier.RatHandle(imgfile)
    outRats.img = inRats.img
    controls.setBlockLength(5)

    ratapplier.apply(myFunc, inRats, outRats, controls=controls)

    col = rat.readColumn(imgfile, 'Value')
    colSqrd = rat.readColumn(imgfile, 'sqrd')
    ok = True
    if (col**2 != colSqrd).any():
        riostestutils.report(TESTNAME, "sqrd incorrect, in sameFile output")
        ok = False
    return ok
コード例 #2
0
def testNewRat(imgfile4):
    makeTestFile(imgfile4, withRat=False)

    inRats = ratapplier.RatAssociations()
    outRats = ratapplier.RatAssociations()
    controls = ratapplier.RatApplierControls()
    controls.setRowCount(256)

    outRats.outimg = ratapplier.RatHandle(imgfile4)
    controls.setBlockLength(3)

    ratapplier.apply(myFuncNewRat, inRats, outRats, controls=controls)

    col = rat.readColumn(imgfile4, 'newCol')
    colIntended = numpy.arange(256, dtype=numpy.uint32)
    ok = (col == colIntended).all()
    if not ok:
        riostestutils.report(TESTNAME,
                             "New RAT incorrect: %s, %s" % (col, colIntended))
    return ok
コード例 #3
0
def testDifferentOutput(imgfile, imgfile2):
    makeTestFile(imgfile2, withRat=False)

    inRats = ratapplier.RatAssociations()
    outRats = ratapplier.RatAssociations()
    controls = ratapplier.RatApplierControls()

    inRats.img = ratapplier.RatHandle(imgfile)
    outRats.outimg = ratapplier.RatHandle(imgfile2)
    controls.setBlockLength(3)

    ratapplier.apply(myFuncDiffFile, inRats, outRats, controls=controls)

    col = rat.readColumn(imgfile, 'Value')
    colSqrd = rat.readColumn(imgfile2, 'sqrd')
    ok = True
    if (col**2 != colSqrd).any():
        riostestutils.report(TESTNAME,
                             "sqrd incorrect, in differentFile output")
        ok = False
    return ok
コード例 #4
0
def testReduceRat(imgfile, imgfile3):
    """
    This test creates a new output image, with all odd pixel values 
    replaced with the even number above it. The RAT must then be copied across
    with the same reduction performed. In this case, only the even numbered 
    rows are written
    """
    # First we copy the raster, with the reduction of pixel values
    infiles = applier.FilenameAssociations()
    outfiles = applier.FilenameAssociations()
    infiles.inimg = imgfile
    outfiles.outimg = imgfile3
    # Make sure we use a format which actually supports RAT's
    controls = applier.ApplierControls()
    controls.setOutputDriverName('HFA')
    applier.apply(rasterReduceFunc, infiles, outfiles, controls=controls)

    # Now use ratapplier to reduce the RAT
    inRats = ratapplier.RatAssociations()
    outRats = ratapplier.RatAssociations()
    controls = ratapplier.RatApplierControls()

    inRats.img = ratapplier.RatHandle(imgfile)
    outRats.outimg = ratapplier.RatHandle(imgfile3)
    controls.setBlockLength(3)

    ratapplier.apply(ratReduceFunc, inRats, outRats, controls=controls)

    col = rat.readColumn(imgfile, 'Value')
    colEven = col[::2]
    colReduced = rat.readColumn(imgfile3, 'Value')[:len(colEven)]
    ok = True
    if (colEven != colReduced).any():
        riostestutils.report(
            TESTNAME, "Reduced RAT incorrect: %s, %s" % (colEven, colReduced))
        ok = False
    return ok
コード例 #5
0
def classifyWithinRATTiled(clumpsImg,
                           classesIntCol,
                           classesNameCol,
                           variables,
                           classifier=RandomForestClassifier(n_estimators=100,
                                                             max_features=3,
                                                             oob_score=True,
                                                             n_jobs=-1),
                           outColInt="OutClass",
                           outColStr="OutClassName",
                           roiCol=None,
                           roiVal=1,
                           classColours=None,
                           scaleVarsRange=False,
                           justFit=False):
    """
A function which will perform a classification within the RAT using a classifier from scikit-learn using the rios ratapplier interface allowing very large RATs to be processed. 

:param clumpsImg: is the clumps image on which the classification is to be performed
:param classesIntCol: is the column with the training data as int values
:param classesNameCol: is the column with the training data as string class names
:param variables: is an array of column names which are to be used for the classification
:param classifier: is an instance of a scikit-learn classifier (e.g., RandomForests which is Default)
:param outColInt: is the output column name for the int class representation (Default: 'OutClass')
:param outColStr: is the output column name for the class names column (Default: 'OutClassName')
:param roiCol: is a column name for a column which specifies the region to be classified. If None ignored (Default: None)
:param roiVal: is a int value used within the roiCol to select a region to be classified (Default: 1)
:param classColours: is a python dict using the class name as the key along with arrays of length 3 specifying the RGB colours for the class.
:param scaleVarsRange: will rescale each variable independently to a range of 0-1 (default: False).
:param justFit: is a boolean specifying that the classifier should just be fitted to the data and not applied (Default: False; i.e., apply classification)


Example::

    from sklearn.ensemble import ExtraTreesClassifier
    from rsgislib.classification import classratutils
    
    classifier = ExtraTreesClassifier(n_estimators=100, max_features=3, n_jobs=-1, verbose=0)
    
    classColours = dict()
    classColours['Forest'] = [0,138,0]
    classColours['NonForest'] = [200,200,200]
    
    variables = ['GreenAvg', 'RedAvg', 'NIR1Avg', 'NIR2Avg', 'NDVI']
    classifyWithinRATTiled(clumpsImg, classesIntCol, classesNameCol, variables, classifier=classifier, classColours=classColours)
        
    # With using range scaling.
    classifyWithinRATTiled(clumpsImg, classesIntCol, classesNameCol, variables, classifier=classifier, classColours=classColours, scaleVarsRange=True)

"""
    # Check gdal is available
    if not haveGDALPy:
        raise Exception(
            "The GDAL python bindings required for this function could not be imported\n\t"
            + gdalErr)
    # Check numpy is available
    if not haveNumpy:
        raise Exception(
            "The numpy module is required for this function could not be imported\n\t"
            + numErr)
    # Check rios rat is available
    if not haveRIOSRat:
        raise Exception(
            "The RIOS rat tools are required for this function could not be imported\n\t"
            + riosRatErr)
    # Check scikit-learn RF is available
    if not haveSKLearnRF:
        raise Exception(
            "The scikit-learn random forests tools are required for this function could not be imported\n\t"
            + sklearnRFErr)
    # Check scikit-learn pre-processing is available
    if not haveSKLearnPreProcess:
        raise Exception(
            "The scikit-learn pre-processing tools are required for this function could not be imported\n\t"
            + sklearnPreProcessErr)

    ratDataset = gdal.Open(clumpsImg, gdal.GA_Update)

    # Read in training classes
    classesInt = rat.readColumn(ratDataset, classesIntCol)
    classesStr = rat.readColumn(ratDataset, classesNameCol)
    ratDataset = None

    validClassStr = classesStr[classesInt > 0]
    validClassInt = classesInt[classesInt > 0]

    #print(validClassInt.shape)
    classNames = numpy.unique(validClassStr)
    classes = numpy.zeros_like(classNames, dtype=numpy.int16)

    i = 0
    classNameIDs = dict()
    for className in classNames:
        classNameStr = str(className.decode())
        if not classNameStr is '':
            #print(validClassInt[validClassStr == className])
            classes[i] = validClassInt[validClassStr == className][0]
            classNameIDs[classNameStr] = classes[i]
            #print("Class \'" + classNameStr + "\' has numerical " + str(classes[i]))
            i = i + 1

    trainLen = validClassInt.shape[0]
    numVars = len(variables)

    #print("Create numpy {} x {} array for training".format(trainLen, numVars))

    trainData = numpy.zeros((trainLen, numVars), numpy.float64)

    in_rats = ratapplier.RatAssociations()
    out_rats = ratapplier.RatAssociations()
    in_rats.inrat = ratapplier.RatHandle(clumpsImg)

    otherargs = ratapplier.OtherArguments()
    otherargs.vars = variables
    otherargs.classIntCol = classesIntCol
    otherargs.trainData = trainData
    otherargs.trainDataOff = 0

    print("Extract Training Data")
    ratapplier.apply(_extractTrainDataFromRAT,
                     in_rats,
                     out_rats,
                     otherargs=otherargs,
                     controls=None)
    print("100%")

    validClassInt = validClassInt[numpy.isfinite(trainData).all(axis=1)]
    validClassStr = validClassStr[numpy.isfinite(trainData).all(axis=1)]
    trainData = trainData[numpy.isfinite(trainData).all(axis=1)]

    print("Training data size: {} x {}".format(trainData.shape[0],
                                               trainData.shape[1]))

    print('Training Classifier')
    classifier.fit(trainData, validClassInt)
    print("Completed")

    print('Calc Classifier Accuracy')
    accVal = classifier.score(trainData, validClassInt)
    print('Classifier Score = {}'.format(round(accVal * 100, 2)))

    if not justFit:
        print("Apply Classifier")
        in_rats = ratapplier.RatAssociations()
        out_rats = ratapplier.RatAssociations()
        in_rats.inrat = ratapplier.RatHandle(clumpsImg)
        out_rats.outrat = ratapplier.RatHandle(clumpsImg)

        otherargs = ratapplier.OtherArguments()
        otherargs.vars = variables
        otherargs.classifier = classifier
        otherargs.outColInt = outColInt
        otherargs.outColStr = outColStr
        otherargs.roiCol = roiCol
        otherargs.roiVal = roiVal
        otherargs.classColours = classColours
        otherargs.classNameIDs = classNameIDs

        ratapplier.apply(_applyClassifier,
                         in_rats,
                         out_rats,
                         otherargs=otherargs,
                         controls=None)
        print("100%")
コード例 #6
0
#!/usr/bin/env python

import argparse
from rios import ratapplier
import numpy


def classifyRAT(info, inputs, outputs):

    outClass = numpy.zeros_like(inputs.inrat.b1Mean, dtype=numpy.int16)

    # Check for loss of water bodies (1)
    outClass = numpy.where(inputs.inrat.b1Mean < 1000, 1, outClass)

    # Save output column to RAT
    outputs.outrat.outClass = outClass


# Set up options
parser = argparse.ArgumentParser()
parser.add_argument("inclumps", nargs=1, type=str, help="Input clumps file")
args = parser.parse_args()

inRats = ratapplier.RatAssociations()
outRats = ratapplier.RatAssociations()

inRats.inrat = ratapplier.RatHandle(args.inclumps[0])
outRats.outrat = ratapplier.RatHandle(args.inclumps[0])

ratapplier.apply(classifyRAT, inRats, outRats)
コード例 #7
0
    def perform_analysis(self, scn_db_obj, sen_obj, plgin_objs):
        logger.info("Processing Scene: {}".format(scn_db_obj.PID))
        if scn_db_obj.Invalid:
            return False, None, False

        rsgis_utils = rsgislib.RSGISPyUtils()
        eodd_utils = EODataDownUtils()

        success = True
        outputs = False
        out_dict = None

        if 'GenChngSummaryFeats' in plgin_objs:
            if plgin_objs['GenChngSummaryFeats'].Completed and plgin_objs[
                    'GenChngSummaryFeats'].Outputs and plgin_objs[
                        'GenChngSummaryFeats'].Success:
                scn_chng_info = plgin_objs['GenChngSummaryFeats'].ExtendedInfo

                scn_unq_name = sen_obj.get_scn_unq_name_record(scn_db_obj)
                out_vec_file = os.path.join(
                    self.params['outvecdir'],
                    "{}_chng_vec.gpkg".format(scn_unq_name))
                if os.path.exists(out_vec_file):
                    delete_vector_file(out_vec_file)

                if sen_obj.get_sensor_name() == 'LandsatGOOG':
                    scn_obs_date = scn_db_obj.Sensing_Time
                elif sen_obj.get_sensor_name() == 'Sentinel2GOOG':
                    scn_obs_date = scn_db_obj.Sensing_Time
                elif sen_obj.get_sensor_name() == 'Sentinel1ASF':
                    scn_obs_date = scn_db_obj.Acquisition_Date
                else:
                    raise Exception("Did not recognise the sensor name...")

                start_date = datetime.datetime(year=2019, month=4, day=30)
                if scn_obs_date > start_date:
                    try:
                        import tqdm
                        progress_bar = rsgislib.TQDMProgressBar()
                    except:
                        from rios import cuiprogress
                        progress_bar = cuiprogress.GDALProgressBar()

                    drv = gdal.GetDriverByName("GPKG")
                    if drv is None:
                        raise Exception("Driver GPKG is not avaiable.")

                    ds = drv.Create(out_vec_file, 0, 0, 0, gdal.GDT_Unknown)
                    if ds is None:
                        raise Exception(
                            "Could not create output file: {}.".format(
                                out_vec_file))

                    out_dict = dict()
                    for tile in scn_chng_info:
                        logger.debug("Processing tile {}...".format(tile))
                        clumps_img = scn_chng_info[tile]

                        in_rats = ratapplier.RatAssociations()
                        out_rats = ratapplier.RatAssociations()
                        in_rats.inrat = ratapplier.RatHandle(clumps_img)

                        lyr = ds.CreateLayer(tile, None, ogr.wkbPoint)
                        if lyr is None:
                            raise Exception(
                                "Could not create output layer: {}.".format(
                                    tile))

                        field_uid_defn = ogr.FieldDefn("uid", ogr.OFTInteger)
                        if lyr.CreateField(field_uid_defn) != 0:
                            raise Exception("Could not create field: 'uid'.")

                        field_prop_chng_defn = ogr.FieldDefn(
                            "prop_chng", ogr.OFTReal)
                        if lyr.CreateField(field_prop_chng_defn) != 0:
                            raise Exception(
                                "Could not create field: 'prop_chng'.")

                        field_score_defn = ogr.FieldDefn(
                            "score", ogr.OFTInteger)
                        if lyr.CreateField(field_score_defn) != 0:
                            raise Exception("Could not create field: 'score'.")

                        # First Observation Date
                        field_firstobsday_defn = ogr.FieldDefn(
                            "firstobsday", ogr.OFTInteger)
                        if lyr.CreateField(field_firstobsday_defn) != 0:
                            raise Exception(
                                "Could not create field: 'firstobsday'.")

                        field_firstobsmonth_defn = ogr.FieldDefn(
                            "firstobsmonth", ogr.OFTInteger)
                        if lyr.CreateField(field_firstobsmonth_defn) != 0:
                            raise Exception(
                                "Could not create field: 'firstobsmonth'.")

                        field_firstobsyear_defn = ogr.FieldDefn(
                            "firstobsyear", ogr.OFTInteger)
                        if lyr.CreateField(field_firstobsyear_defn) != 0:
                            raise Exception(
                                "Could not create field: 'firstobsyear'.")

                        # Last Observation Date
                        field_lastobsday_defn = ogr.FieldDefn(
                            "lastobsday", ogr.OFTInteger)
                        if lyr.CreateField(field_lastobsday_defn) != 0:
                            raise Exception(
                                "Could not create field: 'lastobsday'.")

                        field_lastobsmonth_defn = ogr.FieldDefn(
                            "lastobsmonth", ogr.OFTInteger)
                        if lyr.CreateField(field_lastobsmonth_defn) != 0:
                            raise Exception(
                                "Could not create field: 'lastobsmonth'.")

                        field_lastobsyear_defn = ogr.FieldDefn(
                            "lastobsyear", ogr.OFTInteger)
                        if lyr.CreateField(field_lastobsyear_defn) != 0:
                            raise Exception(
                                "Could not create field: 'lastobsyear'.")

                        # Observation Date Where Score Reached 5
                        field_scr5obsday_defn = ogr.FieldDefn(
                            "scr5obsday", ogr.OFTInteger)
                        if lyr.CreateField(field_scr5obsday_defn) != 0:
                            raise Exception(
                                "Could not create field: 'scr5obsday'.")

                        field_scr5obsmonth_defn = ogr.FieldDefn(
                            "scr5obsmonth", ogr.OFTInteger)
                        if lyr.CreateField(field_scr5obsmonth_defn) != 0:
                            raise Exception(
                                "Could not create field: 'scr5obsmonth'.")

                        field_scr5obsyear_defn = ogr.FieldDefn(
                            "scr5obsyear", ogr.OFTInteger)
                        if lyr.CreateField(field_scr5obsyear_defn) != 0:
                            raise Exception(
                                "Could not create field: 'scr5obsyear'.")

                        lyr_defn = lyr.GetLayerDefn()

                        otherargs = ratapplier.OtherArguments()
                        otherargs.lyr = lyr
                        otherargs.lyr_defn = lyr_defn

                        ratcontrols = ratapplier.RatApplierControls()
                        ratcontrols.setProgress(progress_bar)
                        ratapplier.apply(_ratapplier_check_string_col_valid,
                                         in_rats,
                                         out_rats,
                                         otherargs,
                                         controls=ratcontrols)

                        # Update (create) the JSON LUT file.
                        lut_file_name = "gmw_{}_lut.json".format(tile)
                        lut_file_path = os.path.join(self.params["outlutdir"],
                                                     lut_file_name)
                        eodd_utils.get_file_lock(lut_file_path,
                                                 sleep_period=1,
                                                 wait_iters=120,
                                                 use_except=True)
                        if os.path.exists(lut_file_path):
                            lut_dict = rsgis_utils.readJSON2Dict(lut_file_path)
                        else:
                            lut_dict = dict()

                        obs_date_iso_str = scn_obs_date.isoformat()
                        lut_dict[obs_date_iso_str] = dict()
                        lut_dict[obs_date_iso_str]["file"] = out_vec_file
                        lut_dict[obs_date_iso_str]["layer"] = tile

                        rsgis_utils.writeDict2JSON(lut_dict, lut_file_path)
                        eodd_utils.release_file_lock(lut_file_path)
                        out_dict[tile] = out_vec_file

                    ds = None
                    outputs = True
                    success = True
                else:
                    logger.debug(
                        "Scene is within window used to mask change outside of range."
                    )
            else:
                logger.debug(
                    "No change features available as outputs from previous steps..."
                )
        else:
            logger.debug(
                "GenChngSummaryFeats was not available so previous step had not run..."
            )

        return success, out_dict, outputs