def getAzimuth(f): coords = ee.Array(f.geometry().coordinates().get(0)).transpose() crdLons = ee.List(coords.toList().get(0)) crdLats = ee.List(coords.toList().get(1)) minLon = crdLons.sort().get(0) maxLon = crdLons.sort().get(-1) minLat = crdLats.sort().get(0) maxLat = crdLats.sort().get(-1) azimuth = ee.Number(crdLons.get(crdLats.indexOf(minLat))).subtract(minLon).atan2(ee.Number(crdLats .get( crdLons.indexOf(minLon))).subtract(minLat)).multiply(180.0 / math.pi).add(180.0) return ee.Feature(ee.Geometry.LineString([crdLons.get(crdLats.indexOf(maxLat)), maxLat, minLon, crdLats.get(crdLons.indexOf(minLon))]), {'azimuth': azimuth}).copyProperties(f)
def otsu_function(histogram): counts = ee.Array(ee.Dictionary(histogram).get('histogram')) means = ee.Array(ee.Dictionary(histogram).get('bucketMeans')) size = means.length().get([0]) total = counts.reduce(ee.Reducer.sum(), [0]).get([0]) sums = means.multiply(counts).reduce(ee.Reducer.sum(), [0]).get([0]) mean = sums.divide(total) indices = ee.List.sequence(1, size) #Compute between sum of squares, where each mean partitions the data. def bss_function(i): aCounts = counts.slice(0, 0, i) aCount = aCounts.reduce(ee.Reducer.sum(), [0]).get([0]) aMeans = means.slice(0, 0, i) aMean = aMeans.multiply(aCounts).reduce(ee.Reducer.sum(), [0]).get([0]).divide(aCount) bCount = total.subtract(aCount) bMean = sums.subtract(aCount.multiply(aMean)).divide(bCount) return aCount.multiply(aMean.subtract(mean).pow(2)).add( bCount.multiply(bMean.subtract(mean).pow(2))) bss = indices.map(bss_function) output = means.sort(bss).get([-1]) return output
def testConstructors(self): """Verifies that constructors understand valid parameters.""" from_constant = ee.Image(1) self.assertEqual( ee.ApiFunction.lookup('Image.constant'), from_constant.func) self.assertEqual({'value': 1}, from_constant.args) array_constant = ee.Array([1, 2]) from_array_constant = ee.Image(array_constant) self.assertEqual( ee.ApiFunction.lookup('Image.constant'), from_array_constant.func) self.assertEqual({'value': array_constant}, from_array_constant.args) from_id = ee.Image('abcd') self.assertEqual(ee.ApiFunction.lookup('Image.load'), from_id.func) self.assertEqual({'id': 'abcd'}, from_id.args) from_array = ee.Image([1, 2]) self.assertEqual(ee.ApiFunction.lookup('Image.addBands'), from_array.func) self.assertEqual({ 'dstImg': ee.Image(1), 'srcImg': ee.Image(2) }, from_array.args) from_computed_object = ee.Image(ee.ComputedObject(None, {'x': 'y'})) self.assertEqual({'x': 'y'}, from_computed_object.args) original = ee.Image(1) from_other_image = ee.Image(original) self.assertEqual(from_other_image, original) from_nothing = ee.Image() self.assertEqual(ee.ApiFunction.lookup('Image.mask'), from_nothing.func) self.assertEqual({ 'image': ee.Image(0), 'mask': ee.Image(0) }, from_nothing.args) from_id_and_version = ee.Image('abcd', 123) self.assertEqual( ee.ApiFunction.lookup('Image.load'), from_id_and_version.func) self.assertEqual({'id': 'abcd', 'version': 123}, from_id_and_version.args) from_variable = ee.Image(ee.CustomFunction.variable(None, 'foo')) self.assertTrue(isinstance(from_variable, ee.Image)) self.assertEqual({ 'type': 'ArgumentRef', 'value': 'foo' }, from_variable.encode(None))
def smap_ts_image(img): # scale (int) Default: 30 scale = 30 # extract date from the image dateinfo = ee.Date( img.get("system:time_start")).format("YYYY-MM-dd") # reduce the region to a list, can be configured as per requirements img = img.reduceRegion( reducer=ee.Reducer.toList(), geometry=area, maxPixels=1e8, scale=scale, ) # store data in an ee.Array ssm = ee.Array(img.get("ssm")) susm = ee.Array(img.get("susm")) smp = ee.Array(img.get("smp")) ssma = ee.Array(img.get("ssma")) susma = ee.Array(img.get("susma")) tmpfeature = (ee.Feature(ee.Geometry.Point([0, 0])).set( "ssm", ssm).set("susm", susm).set("smp", smp).set( "ssma", ssma).set("susma", susma).set("dateinfo", dateinfo)) return tmpfeature
def getSegmentImage(self, segmentIndex): mask = self.getSegmentIndexes().eq(segmentIndex.unmask(-1)) image1D = self.segmentsImage.select(self.bandNames1D()) \ .arrayMask(mask) image1D = image1D.mask(image1D.select(0).arrayLength(0).unmask(0)) \ .arrayProject([0]) \ .arrayGet([0]) image2D = self.segmentsImage.select(self.bandNames2D()) \ .arrayMask(mask.toArray(1).unmask(ee.Array([[]], ee.PixelType.double()))) image2D = image2D.mask(image2D.select(0).arrayLength(0).unmask(0)) \ .arrayProject([1]) return image1D \ .addBands(image2D)
def buildMagnitude(fit, nSegments, bandList): segmentTag = buildSegmentTag(nSegments) zeros = ee.Image(ee.Array(ee.List.repeat(0, nSegments))) # Pad zeroes for pixels that have less than 6 segments and then slice the first 6 values def retrieveMags(band): def magsBuilder(x): return ee.String(x).cat('_').cat(band).cat('_MAG') magImg = fit.select(band + '_magnitude').arrayCat( zeros, 0).float().arraySlice(0, 0, nSegments) tags = segmentTag.map(magsBuilder) return magImg.arrayFlatten([tags]) return ee.Image(bandList.map(retrieveMags))
def tasseled_cap(image): coefficients = ee.Array( [[0.3037, 0.2793, 0.4743, 0.5585, 0.5082, 0.1863], [-0.2848, -0.2435, -0.5436, 0.7243, 0.0840, -0.1800], [0.1509, 0.1973, 0.3279, 0.3406, -0.7112, -0.4572], [-0.8242, 0.0849, 0.4392, -0.0580, 0.2012, -0.2768], [-0.3280, 0.0549, 0.1075, 0.1855, -0.4357, 0.8085], [0.1084, -0.9022, 0.4120, 0.0573, -0.0251, 0.0238]]) arrayImage1D = image.select(optical_bands).divide(10000).toArray() arrayImage2D = arrayImage1D.toArray(1) return image.addBands( ee.Image(coefficients).matrixMultiply(arrayImage2D).arrayProject( [0]).arrayFlatten([tasseled_cap_bands]).multiply(10000).int16())
def TrainRBFKernelLocal(self, lmbda=.1, gamma=.5, sampling_factor=1./100., with_task=False, with_cross_validation=False): """ Fit Kernelized RBF Ridge Regression to the current image subsampled. It downloads the data to fit. It uses sklearn.kernel_ridge library. :param lmbda: regularization factor :param gamma: gamma factor to build the kernel (https://en.wikipedia.org/wiki/Radial_basis_function_kernel) :param sampling_factor: percentage of pixels to sample to build the model :param with_cross_validation: donwload the data to fit the model with a task :param with_task: donwload the data to fit the model with a task :return: """ best_params = None if not with_cross_validation: best_params = {"alpha": lmbda, "gamma":gamma} modelo = model_sklearn.KRRModel(best_params=best_params,verbose=1) self._BuildDataSet(sampling_factor, normalize=False) ds_total = converters.eeFeatureCollectionToPandas(self.datos, self.bands_modeling_estimation+["weight"], with_task=with_task) logger.info("Size of downloaded ds: {}".format(ds_total.shape)) output_mean,output_std = model_sklearn.fit_model_local(ds_total,modelo, self.bands_modeling_estimation_input, self.bands_modeling_estimation_output) if with_cross_validation: logger.info("Best params: {}".format(modelo.named_steps["randomizedsearchcv"].best_params_)) ridge_kernel = modelo.named_steps["randomizedsearchcv"].best_estimator_ else: ridge_kernel = modelo.named_steps["kernelridge"] # Copy model parameters self.gamma = ridge_kernel.gamma self.lmbda = ridge_kernel.alpha self.alpha = ee.Array(ridge_kernel.dual_coef_.tolist()) # column vector self.inputs_krr =ee.Array(ridge_kernel.X_fit_.tolist()) self.distance_krr = kernel.RBFDistance(self.gamma) self.kernel_rbf = kernel.Kernel(self.inputs, self.bands_modeling_estimation_input, self.distance_krr, weight_property="weight") # Copy normalization stuff self.inputs_mean = ee.Array(modelo.named_steps["standardscaler"].mean_.tolist()) self.inputs_std = ee.Array(modelo.named_steps["standardscaler"].scale_.tolist()) self.outputs_mean = ee.Array(output_mean.tolist()) self.outputs_std = ee.Array(output_std.tolist()) return
def addFisher(image): # FYI Fisher index optimum threshold 0.63 # Define an Array of Tasseled Cap coefficients. coefficients = ee.Array([[1.7204, 171, 3, -70, -45, -71]]) # Make an Array Image, with a 1-D Array per pixel. constant = ee.Image(1) arrayImage1D = image.addBands(constant).select( ['constant', 'green', 'red', 'nir', 'swir1', 'swir2']).toArray() # Make an Array Image with a 2-D Array per pixel, 6x1. arrayImage2D = arrayImage1D.toArray(1) # Do a matrix multiplication: 6x6 times 6x1. componentsImage = ee.Image(coefficients) \ .matrixMultiply(arrayImage2D) \ .arrayProject([0]) \ .arrayFlatten([['Fisher']]) return image.addBands(componentsImage.rename('Fisher'))
def eeFeatureCollectionToeeArray(ftcol, columns): """ (ee.FeatureCollection, List[str], int) -> ee.Array :param ftcol: :type ftcol: ee.FeatureCollection. :param columns: List[str] colums from the ftcol to retrieve :return: ee.Array """ num_rows = ee.Number(ftcol.size()) bands_to_predict_as_array = ftcol.map( lambda ftr: ee.Feature(None, {"lista": ftr.toArray(columns).toList()})) def extractFromFeature(ftr): ftr = ee.Feature(ftr) return ftr.get("lista") return ee.Array(bands_to_predict_as_array.toList(num_rows).map(extractFromFeature))
def radcal(current, prev): ''' iterator function for orthogonal regression and interactive radiometric normalization ''' k = ee.Number(current) prev = ee.Dictionary(prev) # image is concatenation of reference and target image = ee.Image(prev.get('image')) ncmask = ee.Image(prev.get('ncmask')) nbands = ee.Number(prev.get('nbands')) scale = ee.Number(prev.get('scale')) rect = ee.Geometry(prev.get('rect')) coeffs = ee.List(prev.get('coeffs')) normalized = ee.Image(prev.get('normalized')) # orthoregress reference onto target image1 = image.clip(rect).select(k.add(nbands), k).updateMask(ncmask).rename(['x', 'y']) means = image1.reduceRegion(ee.Reducer.mean(), scale=scale, maxPixels=1e9) \ .toArray()\ .project([0]) Xm = means.get([0]) Ym = means.get([1]) S = ee.Array(image1.toArray() \ .reduceRegion(ee.Reducer.covariance(), geometry=rect, scale=scale, maxPixels=1e9) \ .get('array')) # Pearson correlation R = S.get([0, 1]).divide(S.get([0, 0]).multiply(S.get([1, 1])).sqrt()) eivs = S.eigen() e1 = eivs.get([0, 1]) e2 = eivs.get([0, 2]) # slope and intercept b = e2.divide(e1) a = Ym.subtract(b.multiply(Xm)) coeffs = coeffs.add(ee.List([b, a, R])) # normalize kth band in target normalized = normalized.addBands( image.select(k.add(nbands)).multiply(b).add(a)) return ee.Dictionary({ 'image': image, 'ncmask': ncmask, 'nbands': nbands, 'scale': scale, 'rect': rect, 'coeffs': coeffs, 'normalized': normalized })
def covw(centeredImage, weights=None, maxPixels=1e9): '''Return the (weighted) covariance matrix of a centered image''' if weights == None: weights = centeredImage.multiply(0).add(ee.Image.constant(1)) B1 = centeredImage.bandNames().get(0) b1 = weights.bandNames().get(0) sumWeights = ee.Number( weights.reduceRegion(ee.Reducer.sum(), maxPixels=maxPixels).get(b1)) nPixels = ee.Number( centeredImage.reduceRegion(ee.Reducer.count(), maxPixels=maxPixels).get(B1)) # arr = dataArray(centeredImage.multiply(weights.sqrt())) # return arr.matrixTranspose().matrixMultiply(arr).divide(sumWeights) covW = centeredImage \ .multiply(weights.sqrt()) \ .toArray() \ .reduceRegion(ee.Reducer.centeredCovariance(), maxPixels=1e9) \ .get('array') return ee.Array(covW).multiply(nPixels.divide(sumWeights))
def gfsad30_coords(center_lat, center_lon, edge_len): '''This works for generating coordinates within the GFSAD30 dataset''' coord_in_gfsad30_cropland = False print("center_lat, center_lon: ", center_lat, center_lon) area_of_interest = ee.Geometry.Rectangle([center_lon-edge_len/2, center_lat-edge_len/2, center_lon+edge_len/2, center_lat+edge_len/2]) # 10174268870 gfsad30_asset = 'users/ajsohn/GFSAD30'; gfsad_30_IC = ee.ImageCollection(gfsad30_asset).filterBounds(area_of_interest) img_calc_month_dict = gfsad_30_IC.median() img_calc_month2 = img_calc_month_dict.addBands(ee.Image.pixelLonLat()) # print("img_calc_month2") pixelsDict = img_calc_month2.reduceRegion(reducer=ee.Reducer.toList(), geometry=area_of_interest, maxPixels=1e13, scale=300) try: b1_series = pd.Series(np.array((ee.Array(pixelsDict.get("b1")).getInfo())), name='b1') print("b1_series", b1_series.max()) if round(b1_series.max()) == 2: coord_in_gfsad30_cropland = True print("coord_in_gfsad30_cropland: ", coord_in_gfsad30_cropland) except: warnings.warn('Missing satellite data.') return coord_in_gfsad30_cropland
def image_coll_to_np_array(image_collection, lon, lat, radius): # region of interest geometry = ee.Geometry.Rectangle([ lon - radius, lat - radius, lon + radius, lat + radius]) # reduce to image reduce to region of image img_reduced = image_collection.reduce(ee.Reducer.mean()).reduceRegion( reducer=ee.Reducer.toList(), geometry=geometry, scale=1000) # convert to np array ee_array = ee.Array(img_reduced.toArray()) np_array = np.array(ee_array.getInfo()) return np_array
def covarw(image, weights, scale=30, maxPixels=1e9): '''Return the weighted centered image and its weighted covariance matrix''' geometry = image.geometry() bandNames = image.bandNames() N = bandNames.length() weightsImage = image.multiply(ee.Image.constant(0)).add(weights) means = image.addBands(weightsImage) \ .reduceRegion(ee.Reducer.mean().repeat(N).splitWeights(), scale=scale,maxPixels=maxPixels) \ .toArray() \ .project([1]) centered = image.toArray().subtract(means) B1 = centered.bandNames().get(0) b1 = weights.bandNames().get(0) nPixels = ee.Number(centered.reduceRegion(ee.Reducer.count(), scale=scale, maxPixels=maxPixels).get(B1)) sumWeights = ee.Number(weights.reduceRegion(ee.Reducer.sum(),geometry=geometry, scale=scale, maxPixels=maxPixels).get(b1)) covw = centered.multiply(weights.sqrt()) \ .toArray() \ .reduceRegion(ee.Reducer.centeredCovariance(), geometry=geometry, scale=scale, maxPixels=maxPixels) \ .get('array') covw = ee.Array(covw).multiply(nPixels).divide(sumWeights) return (centered.arrayFlatten([bandNames]), covw)
def getLTvertStack(LTresult): emptyArray = [] vertLabels = [] for i in range(1, runParams['maxSegments'] + 2): vertLabels.append("vert_" + str(i)) emptyArray.append(0) zeros = ee.Image(ee.Array([emptyArray, emptyArray, emptyArray])) lbls = [ ['yrs_', 'src_', 'fit_'], vertLabels, ] vmask = LTresult.arraySlice(0, 3, 4) ltVertStack = LTresult.arrayMask(vmask).arraySlice( 0, 0, 3).addBands(zeros).toArray(1).arraySlice( 1, 0, runParams['maxSegments'] + 1).arrayFlatten(lbls, '') return ltVertStack
def buildCoefs(fit, nSegments, bandList): nBands = bandList.length segmentTag = buildSegmentTag(nSegments) bandTag = buildBandTag('coef', bandList) harmonicTag = ['INTP', 'SLP', 'COS', 'SIN', 'COS2', 'SIN2', 'COS3', 'SIN3'] zeros = ee.Image(ee.Array([ee.List.repeat(0, harmonicTag.length) ])).arrayRepeat(0, nSegments) def retrieveCoefs(band): coefImg = fit.select(band + '_coefs').arrayCat(zeros, 0).float().arraySlice( 0, 0, nSegments) def ceofBuilder(x): return ee.String(x).cat('_').cat(band).cat('_coef') tags = segmentTag.map(ceofBuilder) return coefImg.arrayFlatten([tags, harmonicTag]) return ee.Image(bandList.map(retrieveCoefs))
def getPrincipalComponents(centered, scale, region): # Collapse the bands of the image into a 1D array per pixel. arrays = centered.toArray() # Compute the covariance of the bands within the region. covar = arrays.reduceRegion( **{ 'reducer': ee.Reducer.centeredCovariance(), 'geometry': region, 'scale': scale, 'maxPixels': 1e9 }) # Get the 'array' covariance result and cast to an array. # This represents the band-to-band covariance within the region. covarArray = ee.Array(covar.get('array')) # Perform an eigen analysis and slice apart the values and vectors. eigens = covarArray.eigen() # This is a P-length vector of Eigenvalues. eigenValues = eigens.slice(1, 0, 1) # This is a PxP matrix with eigenvectors in rows. eigenVectors = eigens.slice(1, 1) # Convert the array image to 2D arrays for matrix computations. arrayImage = arrays.toArray(1) # Left multiply the image array by the matrix of eigenvectors. principalComponents = ee.Image(eigenVectors).matrixMultiply(arrayImage) # Turn the square roots of the Eigenvalues into a P-band image. sdImage = ee.Image(eigenValues.sqrt()) \ .arrayProject([0]).arrayFlatten([getNewBandNames('sd')]) # Turn the PCs into a P-band image, normalized by SD. return principalComponents \ .arrayProject([0]) \ .arrayFlatten([getNewBandNames('pc')]) \ .divide(sdImage) \
def on_preview_button_clicked(b): global nbands try: w_text.value = 'iteration started, please wait ...\n' # iMAD inputlist = ee.List.sequence(1, w_iterations.value) first = ee.Dictionary({ 'done': ee.Number(0), 'scale': ee.Number(w_scale.value), 'niter': ee.Number(0), 'image': image1.addBands(image2).clip(poly), 'allrhos': [ee.List.sequence(1, nbands)], 'chi2': ee.Image.constant(0), 'MAD': ee.Image.constant(0) }) result = ee.Dictionary(inputlist.iterate(imad, first)) MAD = ee.Image(result.get('MAD')).rename(madnames) niter = ee.Number(result.get('niter')).getInfo() # threshold nbands = MAD.bandNames().length() chi2 = ee.Image(result.get('chi2')).rename(['chi2']) pval = chi2cdf(chi2, nbands).subtract(1).multiply(-1) tst = pval.gt(ee.Image.constant(0.0001)) MAD = MAD.where(tst, ee.Image.constant(0)) allrhos = ee.Array(result.get('allrhos')).toList() txt = 'Canonical correlations: %s \nIterations: %i\n' % (str( allrhos.get(-1).getInfo()), niter) w_text.value += txt if len(m.layers) > 3: m.remove_layer(m.layers[3]) MAD2 = MAD.select(1).rename('b') ps = MAD2.reduceRegion(ee.Reducer.percentile([1, 99])).getInfo() mn = ps['b_p1'] mx = ps['b_p99'] m.add_layer( TileLayer(url=GetTileLayerUrl(MAD2.visualize(min=mn, max=mx)))) except Exception as e: w_text.value = 'Error: %s\n Retry collect/preview or export to assets' % e
def get_cdf(fc, column): def array_to_features(l): return ee.Feature(None, { column: ee.List(l).get(0), "probability": ee.List(l).get(1) }) # Histogram equalization start: histo = ee.Dictionary( fc.reduceColumns( ee.Reducer.histogram(maxBuckets=2**12, ), [column], ).get("histogram")) valsList = ee.List(histo.get("bucketMeans")) freqsList = ee.List(histo.get("histogram")) cdfArray = ee.Array(freqsList).accum(0) total = cdfArray.get([-1]) normalizedCdf = cdfArray.divide(total) array = ee.Array.cat([valsList, normalizedCdf], 1) return ee.FeatureCollection(array.toList().map(array_to_features))
def pca(image, scale=30, nbands=6, maxPixels=1e9): # center the image bandNames = image.bandNames() meanDict = image.reduceRegion(ee.Reducer.mean(), scale=scale, maxPixels=maxPixels) means = ee.Image.constant(meanDict.values(bandNames)) centered = image.subtract(means) # principal components analysis pcNames = ['pc' + str(i + 1) for i in range(nbands)] centered = centered.toArray() covar = centered.reduceRegion(ee.Reducer.centeredCovariance(), scale=scale, maxPixels=maxPixels) covarArray = ee.Array(covar.get('array')) eigens = covarArray.eigen() lambdas = eigens.slice(1, 0, 1) eivs = eigens.slice(1, 1) centered = centered.toArray(1) pcs=ee.Image(eivs).matrixMultiply(centered) \ .arrayProject([0]) \ .arrayFlatten([pcNames]) return (pcs, lambdas)
def __init__(self, feature_collection, properties, distancia=RBFDistance(.5), weight_property=None): assert type(properties) is list, \ "properties should be a python list object" self.num_rows = ee.Number(feature_collection.size()) # Remove weight propery if present if weight_property is not None: properties = list( filter(lambda prop: prop != weight_property, properties)) self.weight_array = converters.eeFeatureCollectionToeeArray( feature_collection, [weight_property]) else: self.weight_array = ee.Array(ee.List.repeat(1, self.num_rows)) self.weight_array = ee.Array.cat([self.weight_array], 1) self.properties = properties assert len(self.properties) > 1, \ "There is no properties in the current collection" # Get rid of extra columns feature_collection = feature_collection.select(properties) self.feature_collection = feature_collection self.kernel_numpy = None # We store in self.distancia the object which implement the distance self.distancia = distancia self.list_collection = feature_collection.toList(self.num_rows)
def getTasseledCap(img): """Function to compute the Tasseled Cap transformation and return an image""" coefficients = ee.Array([ [0.3037, 0.2793, 0.4743, 0.5585, 0.5082, 0.1863], [-0.2848, -0.2435, -0.5436, 0.7243, 0.0840, -0.1800], [0.1509, 0.1973, 0.3279, 0.3406, -0.7112, -0.4572], [-0.8242, 0.0849, 0.4392, -0.0580, 0.2012, -0.2768], [-0.3280, 0.0549, 0.1075, 0.1855, -0.4357, 0.8085], [0.1084, -0.9022, 0.4120, 0.0573, -0.0251, 0.0238] ]); bands=ee.List(['blue','green','red','nir','swir1','swir2']) # Make an Array Image, with a 1-D Array per pixel. arrayImage1D = img.select(bands).toArray() # Make an Array Image with a 2-D Array per pixel, 6x1. arrayImage2D = arrayImage1D.toArray(1) componentsImage = ee.Image(coefficients).matrixMultiply(arrayImage2D).arrayProject([0]).arrayFlatten([['brightness', 'greenness', 'wetness', 'fourth', 'fifth', 'sixth']]).float(); # Get a multi-band image with TC-named bands. return img.addBands(componentsImage);
def on_preview_button_clicked(b): global cmap try: w_text.value = 'iteration started, please wait ...' MAD = ee.Image(result.get('MAD')).rename(madnames) # threshold nbands = MAD.bandNames().length() chi2 = ee.Image(result.get('chi2')).rename(['chi2']) pval = chi2cdf(chi2, nbands).subtract(1).multiply(-1) tst = pval.gt(ee.Image.constant(0.0001)) MAD = MAD.where(tst, ee.Image.constant(0)) allrhos = ee.Array(result.get('allrhos')).toList() txt = 'Canonical correlations: %s \n' % str(allrhos.get(-1).getInfo()) w_text.value = txt if len(m.layers) > 1: m.remove_layer(m.layers[1]) MAD1 = MAD.select(0).rename('b') ps = MAD1.reduceRegion(ee.Reducer.percentile([2, 98])).getInfo() mn = ps['b_p2'] mx = ps['b_p98'] m.add_layer( TileLayer(url=GetTileLayerUrl(MAD1.visualize(min=mn, max=mx)))) except Exception as e: w_text.value = 'Error: %s\n Retry run/preview or export to assets' % e
def calc_image_pca(image, region=None, scale=90, max_pixels=1e9): """Principal component analysis decomposition of image bands args: image (ee.Image): image to apply pca to region (ee.Geometry | None, optional): region to sample values for covariance matrix, if set to `None` will use img.geometry(). default = None scale (int, optional): scale at which to perform reduction operations, setting higher will prevent OOM errors. default = 90 max_pixels (int, optional): maximum number of pixels to use in reduction operations. default = 1e9 returns: ee.Image: principal components scaled by eigen values """ bandNames = image.bandNames() out_band_names = ee.List.sequence(1, bandNames.length()).map( lambda x: ee.String("pc_").cat(ee.Number(x).int()) ) # Mean center the data to enable a faster covariance reducer # and an SD stretch of the principal components. meanDict = image.reduceRegion( reducer=ee.Reducer.mean(), geometry=region, scale=scale, maxPixels=max_pixels ) means = ee.Image.constant(meanDict.values(bandNames)) centered = image.subtract(means) # Collapse the bands of the image into a 1D array per pixel. arrays = centered.toArray() # Compute the covariance of the bands within the region. covar = arrays.reduceRegion( reducer=ee.Reducer.centeredCovariance(), geometry=region, scale=scale, maxPixels=max_pixels, ) # Get the 'array' covariance result and cast to an array. # This represents the band-to-band covariance within the region. covarArray = ee.Array(covar.get("array")) # Perform an eigen analysis and slice apart the values and vectors. eigens = covarArray.eigen() # This is a P-length vector of Eigenvalues. eigenValues = eigens.slice(1, 0, 1) # This is a PxP matrix with eigenvectors in rows. eigenVectors = eigens.slice(1, 1) # Convert the array image to 2D arrays for matrix computations. arrayImage = arrays.toArray(1) # Left multiply the image array by the matrix of eigenvectors. principalComponents = ee.Image(eigenVectors).matrixMultiply(arrayImage) # Turn the square roots of the Eigenvalues into a P-band image. sdImage = ( ee.Image(eigenValues.sqrt()).arrayProject([0]).arrayFlatten([out_band_names]) ) # Turn the PCs into a P-band image, normalized by SD. return ( principalComponents # Throw out an an unneeded dimension, [[]] -> []. .arrayProject([0]) # Make the one band array image a multi-band image, [] -> image. .arrayFlatten([out_band_names]) # Normalize the PCs by their SDs. .divide(sdImage) )
def restrend_pointwise(year_start, year_end, geojson, EXECUTION_ID, logger): """Calculate temporal NDVI analysis. Calculates the trend of temporal NDVI using NDVI data from the MODIS Collection 6 MOD13Q1 dataset. Areas where changes are not significant are masked out using a Mann-Kendall test. Args: year_start: The starting year (to define the period the trend is calculated over). year_end: The ending year (to define the period the trend is calculated over). geojson: A polygon defining the area of interest. Returns: Output of google earth engine task. """ # Function to integrate NDVI dataset from 15d to 1yr def int_15d_1yr_clim(img_stack): img_coll = ee.List([]) for k in range(1, 34): ndvi_lyr = img_stack.select( ee.List.sequence((k - 1) * 24, (k * 24) - 1)).reduce( ee.Reducer.mean()).rename(['ndvi']).set({'year': 1981 + k}) img_coll = img_coll.add(ndvi_lyr) return ee.ImageCollection(img_coll) # Function to compute differences between observed and predicted NDVI and comilation in an image collection def stack(year_start, year_end): img_coll = ee.List([]) for k in range(year_start, year_end): ndvi = ndvi_1yr_o.filter(ee.Filter.eq('year', k)).select('ndvi').median() clim = clim_1yr_o.filter(ee.Filter.eq('year', k)).select('ndvi').median() img = ndvi.addBands(clim.addBands(ee.Image(k).float())).rename( ['ndvi', 'clim', 'year']).set({'year': k}) img_coll = img_coll.add(img) return ee.ImageCollection(img_coll) # Function to predict NDVI from climate first = ee.List([]) def ndvi_clim_p(image, list): ndvi = lf_clim_ndvi.select('offset').add( (lf_clim_ndvi.select('scale').multiply(image))).set( {'year': image.get('year')}) return ee.List(list).add(ndvi) # Create image collection of residuals def ndvi_res(year_start, year_end): img_coll = ee.List([]) for k in range(year_start, year_end): ndvi_o = coll_1yr_o.filter(ee.Filter.eq( 'year', k)).select('ndvi').median() ndvi_p = ndvi_1yr_p.filter(ee.Filter.eq('year', k)).median() ndvi_r = ee.Image(k).float().addBands(ndvi_o.subtract(ndvi_p)) img_coll = img_coll.add(ndvi_r.rename(['year', 'ndvi_res'])) return ee.ImageCollection(img_coll) ndvi_1yr_o = preproc.modis_ndvi_annual_integral(year_start, year_end) # TODO: define clim_15d_o which is the merra-2 soil moisture data. For now, # forcing use of Senegal data for testing. clim_15d_o = ee.Image( 'users/geflanddegradation/soil/sen_soilm_merra2_15d_1982_2015') # Apply function to compute climate annual integrals from 15d observed data clim_1yr_o = int_15d_1yr_clim(clim_15d_o.divide(10000)) # Apply function to create image collection with stack of NDVI int, climate int and year coll_1yr_o = stack(year_start, year_end) # Reduce the collection with the linear fit reducer (independent var are followed by dependent var) lf_clim_ndvi = coll_1yr_o.select(['clim', 'ndvi']).reduce(ee.Reducer.linearFit()) # Apply function to predict NDVI based on climate ndvi_1yr_p = ee.ImageCollection( ee.List(coll_1yr_o.select('clim').iterate(ndvi_clim_p, first))) # Apply function to compute NDVI annual residuals ndvi_1yr_r = ndvi_res(year_start, year_end) # Fit a linear regression to the NDVI residuals lf_prest = ndvi_1yr_r.select(['year', 'ndvi_res']).reduce(ee.Reducer.linearFit()) # Compute Kendall statistics mk_prest = stats.mann_kendall(ndvi_1yr_r.select('ndvi_res')) # Define Kendall parameter values for a significance of 0.05 period = year_end - year_start + 1 coefficients = ee.Array([ 4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 31, 33, 36, 40, 43, 47, 50, 54, 59, 63, 66, 70, 75, 79, 84, 88, 93, 97, 102, 106, 111, 115, 120, 126, 131, 137, 142 ]) kendall = coefficients.get([period - 4]) # Create export function export = { 'image': lf_prest.select('scale').where( mk_prest.abs().lte(kendall), -99999).where( lf_prest.select('scale').abs().lte(0.000001), -99999).unmask(-99999), 'description': EXECUTION_ID, 'fileNamePrefix': EXECUTION_ID, 'bucket': BUCKET, 'maxPixels': 10000000000, 'scale': 250, 'region': util.get_coords(geojson) } # Export final mosaic to assets task = ee.batch.Export.image.toCloudStorage(**export) task.start() task_state = task.status().get('state') while task_state == 'READY' or task_state == 'RUNNING': task_progress = task.status().get('progress', 0.0) # update GEF-EXECUTION progress logger.send_progress(task_progress) # update variable to check the condition task_state = task.status().get('state') sleep(5) return "https://{}.storage.googleapis.com/{}.tif".format( BUCKET, EXECUTION_ID)
def l8Correction(img): # Julian Day imgDate_OLI = img.date() FOY_OLI = ee.Date.fromYMD(imgDate_OLI.get('year'), 1, 1) JD_OLI = imgDate_OLI.difference(FOY_OLI, 'day').int().add(1) # ozone DU_OLI = ee.Image(OZONE.filterDate(imgDate_OLI, imgDate_OLI.advance(7,'day')).mean()) # Earth-Sun distance d_OLI = ee.Image.constant(img.get('EARTH_SUN_DISTANCE')) # Sun elevation SunEl_OLI = ee.Image.constant(img.get('SUN_ELEVATION')) # Sun azimuth SunAz_OLI = ee.Image.constant(img.get('SUN_AZIMUTH')) # Satellite zenith SatZe_OLI = ee.Image(0.0) cosdSatZe_OLI = (SatZe_OLI).multiply(PI.divide(ee.Image(180))).cos() sindSatZe_OLI = (SatZe_OLI).multiply(PI.divide(ee.Image(180))).sin() # Satellite azimuth SatAz_OLI = ee.Image(0.0) # Sun zenith SunZe_OLI = ee.Image(90).subtract(SunEl_OLI) cosdSunZe_OLI = SunZe_OLI.multiply(PI.divide(ee.Image.constant(180))).cos() # in degrees sindSunZe_OLI = SunZe_OLI.multiply(PI.divide(ee.Image(180))).sin() # in degrees # Relative azimuth RelAz_OLI = ee.Image(SunAz_OLI) cosdRelAz_OLI = RelAz_OLI.multiply(PI.divide(ee.Image(180))).cos() # Pressure calculation P_OLI = ee.Image(101325).multiply(ee.Image(1).subtract(ee.Image(0.0000225577).multiply(DEM)).pow(5.25588)).multiply( 0.01) Po_OLI = ee.Image(1013.25) # Radiometric Calibration # # define bands to be converted to radiance bands_OLI = ['B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7'] # radiance_mult_bands rad_mult_OLI = ee.Image(ee.Array([ee.Image(img.get('RADIANCE_MULT_BAND_1')), ee.Image(img.get('RADIANCE_MULT_BAND_2')), ee.Image(img.get('RADIANCE_MULT_BAND_3')), ee.Image(img.get('RADIANCE_MULT_BAND_4')), ee.Image(img.get('RADIANCE_MULT_BAND_5')), ee.Image(img.get('RADIANCE_MULT_BAND_6')), ee.Image(img.get('RADIANCE_MULT_BAND_7'))] )).toArray(1) # radiance add band rad_add_OLI = ee.Image(ee.Array([ee.Image(img.get('RADIANCE_ADD_BAND_1')), ee.Image(img.get('RADIANCE_ADD_BAND_2')), ee.Image(img.get('RADIANCE_ADD_BAND_3')), ee.Image(img.get('RADIANCE_ADD_BAND_4')), ee.Image(img.get('RADIANCE_ADD_BAND_5')), ee.Image(img.get('RADIANCE_ADD_BAND_6')), ee.Image(img.get('RADIANCE_ADD_BAND_7'))] )).toArray(1) # create an empty image to save new radiance bands to imgArr_OLI = img.select(bands_OLI).toArray().toArray(1) Ltoa_OLI = imgArr_OLI.multiply(rad_mult_OLI).add(rad_add_OLI) # esun ESUN_OLI = ee.Image.constant(197.24790954589844)\ .addBands(ee.Image.constant(201.98426818847656))\ .addBands(ee.Image.constant(186.12677001953125))\ .addBands(ee.Image.constant(156.95257568359375))\ .addBands(ee.Image.constant(96.04714965820312))\ .addBands(ee.Image.constant(23.8833221450863))\ .addBands(ee.Image.constant(8.04995873449635)).toArray().toArray(1) ESUN_OLI = ESUN_OLI.multiply(ee.Image(1)) ESUNImg_OLI = ESUN_OLI.arrayProject([0]).arrayFlatten([bands_OLI]) # Ozone Correction # # Ozone coefficients koz_OLI = ee.Image.constant(0.0039).addBands(ee.Image.constant(0.0218))\ .addBands(ee.Image.constant(0.1078))\ .addBands(ee.Image.constant(0.0608))\ .addBands(ee.Image.constant(0.0019))\ .addBands(ee.Image.constant(0))\ .addBands(ee.Image.constant(0))\ .toArray().toArray(1) # Calculate ozone optical thickness Toz_OLI = koz_OLI.multiply(DU_OLI).divide(ee.Image.constant(1000)) # Calculate TOA radiance in the absense of ozone Lt_OLI = Ltoa_OLI.multiply(((Toz_OLI)).multiply( (ee.Image.constant(1).divide(cosdSunZe_OLI)).add(ee.Image.constant(1).divide(cosdSatZe_OLI))).exp()) # Rayleigh optical thickness bandCenter_OLI = ee.Image(443).divide(1000).addBands(ee.Image(483).divide(1000))\ .addBands(ee.Image(561).divide(1000))\ .addBands(ee.Image(655).divide(1000))\ .addBands(ee.Image(865).divide(1000))\ .addBands(ee.Image(1609).divide(1000))\ .addBands(ee.Number(2201).divide(1000))\ .toArray().toArray(1) # create an empty image to save new Tr values to Tr_OLI = (P_OLI.divide(Po_OLI)).multiply(ee.Image(0.008569).multiply(bandCenter_OLI.pow(-4))).multiply((ee.Image(1).add( ee.Image(0.0113).multiply(bandCenter_OLI.pow(-2))).add(ee.Image(0.00013).multiply(bandCenter_OLI.pow(-4))))) # Fresnel Reflection # # Specular reflection (s- and p- polarization states) theta_V_OLI = ee.Image(0.0000000001) sin_theta_j_OLI = sindSunZe_OLI.divide(ee.Image(1.333)) theta_j_OLI = sin_theta_j_OLI.asin().multiply(ee.Image(180).divide(PI)) theta_SZ_OLI = SunZe_OLI R_theta_SZ_s_OLI = (((theta_SZ_OLI.multiply(PI.divide(ee.Image(180)))).subtract( theta_j_OLI.multiply(PI.divide(ee.Image(180))))).sin().pow(2)).divide((((theta_SZ_OLI.multiply( PI.divide(ee.Image(180)))).add(theta_j_OLI.multiply(PI.divide(ee.Image(180))))).sin().pow(2))) R_theta_V_s_OLI = ee.Image(0.0000000001) R_theta_SZ_p_OLI = ( ((theta_SZ_OLI.multiply(PI.divide(180))).subtract(theta_j_OLI.multiply(PI.divide(180)))).tan().pow(2)).divide( (((theta_SZ_OLI.multiply(PI.divide(180))).add(theta_j_OLI.multiply(PI.divide(180)))).tan().pow(2))) R_theta_V_p_OLI = ee.Image(0.0000000001) R_theta_SZ_OLI = ee.Image(0.5).multiply(R_theta_SZ_s_OLI.add(R_theta_SZ_p_OLI)) R_theta_V_OLI = ee.Image(0.5).multiply(R_theta_V_s_OLI.add(R_theta_V_p_OLI)) # Rayleigh scattering phase function # # Sun-sensor geometry theta_neg_OLI = ((cosdSunZe_OLI.multiply(ee.Image(-1))).multiply(cosdSatZe_OLI)).subtract( (sindSunZe_OLI).multiply(sindSatZe_OLI).multiply(cosdRelAz_OLI)) theta_neg_inv_OLI = theta_neg_OLI.acos().multiply(ee.Image(180).divide(PI)) theta_pos_OLI = (cosdSunZe_OLI.multiply(cosdSatZe_OLI)).subtract( sindSunZe_OLI.multiply(sindSatZe_OLI).multiply(cosdRelAz_OLI)) theta_pos_inv_OLI = theta_pos_OLI.acos().multiply(ee.Image(180).divide(PI)) cosd_tni_OLI = theta_neg_inv_OLI.multiply(PI.divide(180)).cos() # in degrees cosd_tpi_OLI = theta_pos_inv_OLI.multiply(PI.divide(180)).cos() # in degrees Pr_neg_OLI = ee.Image(0.75).multiply((ee.Image(1).add(cosd_tni_OLI.pow(2)))) Pr_pos_OLI = ee.Image(0.75).multiply((ee.Image(1).add(cosd_tpi_OLI.pow(2)))) # Rayleigh scattering phase function Pr_OLI = Pr_neg_OLI.add((R_theta_SZ_OLI.add(R_theta_V_OLI)).multiply(Pr_pos_OLI)) # Calulate Lr, denom_OLI = ee.Image(4).multiply(PI).multiply(cosdSatZe_OLI) Lr_OLI = (ESUN_OLI.multiply(Tr_OLI)).multiply(Pr_OLI.divide(denom_OLI)) # Rayleigh corrected radiance Lrc_OLI = (Lt_OLI.divide(ee.Image(10))).subtract(Lr_OLI) LrcImg_OLI = Lrc_OLI.arrayProject([0]).arrayFlatten([bands_OLI]) # Rayleigh corrected reflectance prc_OLI = Lrc_OLI.multiply(PI).multiply(d_OLI.pow(2)).divide(ESUN_OLI.multiply(cosdSunZe_OLI)) prcImg_OLI = prc_OLI.arrayProject([0]).arrayFlatten([bands_OLI]) # Aerosol Correction # # Bands in nm bands_nm_OLI = ee.Image(443).addBands(ee.Image(483))\ .addBands(ee.Image(561))\ .addBands(ee.Image(655))\ .addBands(ee.Image(865))\ .addBands(ee.Image(0))\ .addBands(ee.Image(0))\ .toArray().toArray(1) # Lam in SWIR bands Lam_6_OLI = LrcImg_OLI.select('B6') Lam_7_OLI = LrcImg_OLI.select('B7') # Calculate aerosol type eps_OLI = (((((Lam_7_OLI).divide(ESUNImg_OLI.select('B7'))).log()).subtract( ((Lam_6_OLI).divide(ESUNImg_OLI.select('B6'))).log())).divide(ee.Image(2201).subtract(ee.Image(1609)))) # Calculate multiple scattering of aerosols for each band Lam_OLI = (Lam_7_OLI).multiply(((ESUN_OLI).divide(ESUNImg_OLI.select('B7')))).multiply( (eps_OLI.multiply(ee.Image(-1))).multiply((bands_nm_OLI.divide(ee.Image(2201)))).exp()) # diffuse transmittance trans_OLI = Tr_OLI.multiply(ee.Image(-1)).divide(ee.Image(2)).multiply(ee.Image(1).divide(cosdSatZe_OLI)).exp() # Compute water-leaving radiance Lw_OLI = Lrc_OLI.subtract(Lam_OLI).divide(trans_OLI) # water-leaving reflectance pw_OLI = (Lw_OLI.multiply(PI).multiply(d_OLI.pow(2)).divide(ESUN_OLI.multiply(cosdSunZe_OLI))) pwImg_OLI = pw_OLI.arrayProject([0]).arrayFlatten([bands_OLI]) # Rrs Rrs_coll = (pw_OLI.divide(PI).arrayProject([0]).arrayFlatten([bands_OLI]).slice(0, 5)) # final processing for masking to get clear water pixels # tile geometry footprint = img.geometry() # cloud mask scsmask = ee.Algorithms.Landsat.simpleCloudScore(ee.Algorithms.Landsat.TOA(img)).select('cloud').lt(10) qamask = extractBits(img.select('BQA'),4,4,'clouds').eq(0) # from qa band cloudmask = scsmask.And(qamask) # water mask watermask = Rrs_coll.normalizedDifference(['B3','B5']).gt(0) return ee.Image(Rrs_coll).clip(footprint).updateMask(cloudmask.And(watermask)).set('system:time_start', img.get('system:time_start'))
def dataArray(image, maxPixels=1e9): '''Return image as a data array''' nBands = image.bandNames().length() return ee.Array(image \ .reduceRegion(ee.Reducer.toList(nBands), maxPixels=maxPixels) \ .get('list'))
def imad(current, prev): import numpy as np image1 = ee.Image(ee.Dictionary(current).get('image1')) image2 = ee.Image(ee.Dictionary(current).get('image2')) weights = ee.Image(ee.Dictionary(prev).get('weights')) region = image1.geometry() bNames1 = image1.bandNames() bNames2 = image2.bandNames() nBands = bNames1.length() centeredImage1 = centerw(image1, weights) centeredImage2 = centerw(image2, weights) centeredImage = ee.Image.cat(centeredImage1, centeredImage2) covarArray = covw(centeredImage, weights) s11 = covarArray.slice(0, 0, nBands).slice(1, 0, nBands) s22 = covarArray.slice(0, nBands).slice(1, nBands) s12 = covarArray.slice(0, 0, nBands).slice(1, nBands) s21 = covarArray.slice(0, nBands).slice(1, 0, nBands) c1 = s12.matrixMultiply(s22.matrixInverse()).matrixMultiply(s21) b1 = s11 c2 = s21.matrixMultiply(s11.matrixInverse()).matrixMultiply(s12) b2 = s22 # solution of generalized eigenproblems rho2, A = geneiv(c1, b1) _, B = geneiv(c2, b2) # canonical correlations and MAD variances rhos = rho2.sqrt().project(ee.List([1])) ones = ee.Array(ee.List.repeat(1, nBands)) twos = ee.Array(ee.List.repeat(2, nBands)) s2 = ones.subtract(rhos).multiply(twos) variance = ee.Image.constant(s2) # ------------------------------------------------------------------- s11 = np.mat(s11.getInfo()) s12 = np.mat(s12.getInfo()) A = np.mat(A.getInfo()) B = np.mat(B.getInfo()) # ensure sum of positive correlations between X and U is positive tmp = np.diag(1 / np.sqrt(np.diag(s11))) s = np.ravel(np.sum(tmp * s11 * A, axis=0)) A = A * np.diag(s / np.abs(s)) # ensure positive correlation tmp = np.diag(A.T * s12 * B) B = B * np.diag(tmp / abs(tmp)) # ------------------------------------------------------------------- # canonical and MAD variates Arr = ee.Array(A.tolist()) Brr = ee.Array(B.tolist()) centeredImage1Array = centeredImage1.toArray().toArray(1) centeredImage2Array = centeredImage2.toArray().toArray(1) U = ee.Image(Arr).matrixMultiply(centeredImage1Array) \ .arrayProject([0]) \ .arrayFlatten([bNames1]) V = ee.Image(Brr).matrixMultiply(centeredImage2Array) \ .arrayProject([0]) \ .arrayFlatten([bNames2]) MAD = U.subtract(V) # chi square image chi2 = (MAD.pow(2)) \ .divide(variance) \ .reduce(ee.Reducer.sum()) \ .clip(region) # no-change probability weights = ee.Image.constant(1.0).subtract(chi2cdf(chi2, 6.0)) return ee.Dictionary({'weights': weights, 'MAD': MAD})
# Array-based spectral unmixing. # Create a mosaic of Landsat 5 images from June through September, 2007. allBandMosaic = ee.ImageCollection('LANDSAT/LT05/C01/T1') \ .filterDate('2007-06-01', '2007-09-30') \ .select('B[0-7]') \ .median() # Create some representative endmembers computed previously by sampling # the Landsat 5 mosaic. urbanEndmember = [88, 42, 48, 38, 86, 115, 59] vegEndmember = [50, 21, 20, 35, 50, 110, 23] waterEndmember = [51, 20, 14, 9, 7, 116, 4] # Compute the 3x7 pseudo inverse. endmembers = ee.Array([urbanEndmember, vegEndmember, waterEndmember]) inverse = ee.Image(endmembers.matrixPseudoInverse().transpose()) # Convert the bands to a 2D 7x1 array. The toArray() call concatenates # pixels from each band along the default axis 0 into a 1D vector per # pixel, and the toArray(1) call concatenates each band (in this case # just the one band of 1D vectors) along axis 1, forming a 2D array. inputValues = allBandMosaic.toArray().toArray(1) # Matrix multiply the pseudo inverse of the endmembers by the pixels to # get a 3x1 set of endmembers fractions from 0 to 1. unmixed = inverse.matrixMultiply(inputValues) # Create and show a colored image of the endmember fractions. Since we know # the result has size 3x1, project down to 1D vectors at each pixel (since the # second axis is pointless now), and then flatten back to a regular scalar