def rasterize(fc: ee.FeatureCollection, layer_name: str) -> ee.Image:
fc = fc.map(lambda f: ee.Feature(f).set({layer_name: 1}))
img = fc.reduceToImage([layer_name], ee.Reducer.first()) \
.unmask() \
.float() \
.rename(layer_name)
return img
def find_field_cluster_asynchronous(field_collection: ee.FeatureCollection) -> ee.Number:
"""Asynchronous lookup for the fields number.
This function uses Earth Engine means of finding the number.
"""
clusters = field_collection.iterate(lambda elem, prev: ee.List(prev).add(elem.get("cluster")), ee.List([]))
return ee.Number(ee.List(clusters).reduce(ee.Reducer.mode())).int8()
def tfexporter(collection: ee.FeatureCollection,
export: str,
prefix: str,
fname: str,
selectors: Optional[ee.List] = None,
dropselectors: Optional[ee.List] = None,
bucket: Optional[str] = None) -> ee.batch.Task:
'''Creates and starts a task to export a ee.FeatureCollection to a TFRecord
file in Google Drive or Google Cloud Storage (GCS).
GCS: gs://bucket/prefix/fname.tfrecord
Drive: prefix/fname.tfrecord
Args
- collection: ee.FeatureCollection
- export: str, 'drive' for Drive, 'gcs' for GCS
- prefix: str, folder name in Drive or GCS to export to, no trailing '/'
- fname: str, filename
- selectors: None or ee.List of str, names of properties to include in
output, set to None to include all properties
- dropselectors: None or ee.List of str, names of properties to exclude
- bucket: None or str, name of GCS bucket, only used if export=='gcs'
Returns
- task: ee.batch.Task
'''
if dropselectors is not None:
if selectors is None:
selectors = collection.first().propertyNames()
selectors = selectors.removeAll(dropselectors)
if export == 'gcs':
task = ee.batch.Export.table.toCloudStorage(
collection=collection,
description=fname,
bucket=bucket,
fileNamePrefix=f'{prefix}/{fname}',
fileFormat='TFRecord',
selectors=selectors)
elif export == 'drive':
task = ee.batch.Export.table.toDrive(collection=collection,
description=fname,
folder=prefix,
fileNamePrefix=fname,
fileFormat='TFRecord',
selectors=selectors)
else:
raise ValueError(f'export "{export}" is not one of ["gcs", "drive"]')
task.start()
return task
def get_array_patches(img: ee.Image,
scale: Numeric,
ksize: Numeric,
points: ee.FeatureCollection,
export: str,
prefix: str,
fname: str,
selectors: Optional[ee.List] = None,
dropselectors: Optional[ee.List] = None,
bucket: Optional[str] = None) -> ee.batch.Task:
'''Creates and starts a task to export square image patches in TFRecord
format to Google Drive or Google Cloud Storage (GCS). The image patches are
sampled from the given ee.Image at specific coordinates.
Args
- img: ee.Image, image covering the entire region of interest
- scale: int or float, scale in meters of the projection to sample in
- ksize: int or float, radius of square image patch
- points: ee.FeatureCollection, coordinates from which to sample patches
- export: str, 'drive' for Google Drive, 'gcs' for GCS
- prefix: str, folder name in Drive or GCS to export to, no trailing '/'
- fname: str, filename for export
- selectors: None or ee.List, names of properties to include in output,
set to None to include all properties
- dropselectors: None or ee.List, names of properties to exclude
- bucket: None or str, name of GCS bucket, only used if export=='gcs'
Returns: ee.batch.Task
'''
kern = ee.Kernel.square(radius=ksize, units='pixels')
patches_array = img.neighborhoodToArray(kern)
# ee.Image.sampleRegions() does not cut it for larger collections,
# using mapped sample instead
samples = points.map(lambda pt: sample_patch(pt, patches_array, scale))
# export to a TFRecord file which can be loaded directly in TensorFlow
return tfexporter(collection=samples,
export=export,
prefix=prefix,
fname=fname,
selectors=selectors,
dropselectors=dropselectors,
bucket=bucket)
def features(self, land: Land = None) -> FeatureCollection:
if land:
return FeatureCollection(self.feature_dict[land.value])
return FeatureCollection([*self.feature_dict[Land.Lowland.value], *self.feature_dict[Land.Highland.value]])
def TOC_Feature_coor(featurecollection_input: ee.FeatureCollection,
QCname: str,
Indexname: str,
thresholdList: ee.List,
Classname: str = None,
ClassList: dict = None,
exportCoor: str = False,
exportVariable: str = False) -> None:
'''
This function is to export the coordinates of the TOC curve. The input format is the ee.FeatureCollection.
:param featurecollection_input: (ee.FeatureCollection) featurecollection that contains reference and index properties
:param QCname: (string) The name of reference property
:param Indexname: (string) The name of index property
:param thresholdList: (ee.List/list) The list of thresholds. (from high to low or from low to high)
:param Classname: (string) The name of class property when applying stratified sampling
:param ClassList: (disctionary) The size of stratums {classname: classsize, classname: classsize ...}
:param exportCoor: (string / default is False) The default parameter is False (not export the coordinates), string should be output path of coordinates. (extension should be txt)
:param exportVariable: (string / default is False) The default parameter is False (not export the coordinates), string should be output of variables. (extension should be txt)
:return: (ee.list) coordinates of the TOC curve. The first column is x, the second is y, the third is threshold of the point.
'''
featurecollection_list = featurecollection_input.toList(
featurecollection_input.size())
def getPropertyIndex(x):
return ee.Feature(x).getNumber(Indexname)
def getPropertyQC(x):
return ee.Feature(x).getNumber(QCname)
Index_1 = featurecollection_list.map(getPropertyIndex)
Index_2 = ee.Array(Index_1)
QC_1 = featurecollection_list.map(getPropertyQC)
QC_2 = ee.Array(QC_1)
if ((Classname is not None) and (ClassList is not None)):
if (type(Classname) is not str):
Classname = Classname.getInfo()
if (type(ClassList) is not ee.Dictionary):
ClassList = ee.Dictionary(ClassList)
classProperty = featurecollection_input.aggregate_histogram(Classname)
def classWeight(key, value):
presenceInClass = classProperty.get(key)
return ee.Number(value).divide(presenceInClass)
ClassWeights = ClassList.map(classWeight)
print(ClassWeights.getInfo())
def setWeight(x):
x = ee.Feature(x)
return ClassWeights.get(x.get(Classname))
weight = ee.Array(featurecollection_list.map(setWeight))
else:
weight = ee.Array(ee.List.repeat(1, QC_1.size()))
presenceInY = sumArray(QC_2.multiply(weight))
totalNumber = sumArray(weight)
sequence = ee.Algorithms.If(
ee.Number(thresholdList.get(0)).lt(thresholdList.get(-1)), 1, 0)
def coordinatelist(x):
index1 = compareIndexArray(Index_2, x, sequence)
index1_weight = index1.multiply(weight)
index2 = ee.Array((index1.add(QC_2).gte(2)))
index2_weight = index2.multiply(weight)
x_1 = sumArray(index1_weight)
y_1 = sumArray(index2_weight)
coor = ee.List([x_1, y_1, ee.Number(x)])
return coor
result = thresholdList.map(coordinatelist)
result_output = result
if (exportCoor):
result = np.array(result.getInfo()).transpose()
exportCoorFunc(exportCoor, np.array([result[0].tolist()]),
np.array([result[1].tolist()]), result[2].tolist())
if (exportVariable):
pInY = presenceInY.getInfo()
tNumber = totalNumber.getInfo()
AUC = calculate_AUC(np.array([result[0].tolist()]),
np.array([result[1].tolist()]), pInY, tNumber)
ccorner = correctCorner(np.array([result[0].tolist()]),
np.array([result[1].tolist()]), pInY)
exportVariableFunc(exportVariable, tNumber, pInY, ccorner, AUC)
return result_output
def classify(self, dataset: ee.FeatureCollection) -> ee.FeatureCollection:
return dataset.classify(self.regressor)
def test_feature_collection_simple_with_where_is_not(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where a is not NULL ')
correct = FeatureCollection('ft:mytable').filter(Filter().neq('a', 'null'))
query = sql2gee._feature_collection
self.assertEqual(query, correct)
return
def test_simple_feature_collection(self):
sql2gee = SQL2GEE('select * from "ft:mytable" ')
query = sql2gee._feature_collection
correct = FeatureCollection('ft:mytable')
self.assertEqual(query, correct)
return
def test_feature_collection_simple_with_where_string(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where a = "a" ')
correct = FeatureCollection('ft:mytable').filter(Filter().eq('a', "a"))
query = sql2gee._feature_collection
self.assertEqual(query, correct)
return
def test_feature_collection_simple_with_where_gt_and_like(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where b > 2 and a like "%2%"')
correct = FeatureCollection('ft:mytable').filter(Filter().And(Filter().gt('b', 2), Filter().stringContains('a', '2')))
query = sql2gee._feature_collection
self.assertEqual(query, correct)
return
def test_feature_collection_simple_with_where_not_like(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where a not like "%2%" ')
correct = FeatureCollection('ft:mytable').filter(Filter().stringContains('a', '2').Not())
query = sql2gee._feature_collection
self.assertEqual(query, correct)
return
def test_feature_collection_simple_with_where_like_endsWith(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where a like "%2" ')
correct = FeatureCollection('ft:mytable').filter(Filter().stringEndsWith('a', '2'))
query = sql2gee._feature_collection
self.assertEqual(query, correct)
return
def test_feature_collection_simple_with_where_not_float(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where not a > 2.2 ')
correct = FeatureCollection('ft:mytable').filter(Filter().gt('a', 2.2).Not())
query = sql2gee._feature_collection
self.assertEqual(query, correct)
return
def test_feature_collection_simple_with_where_several_conditions(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where (a > 2 or b < 2) and b = 2 ')
query = sql2gee._feature_collection
correct = FeatureCollection('ft:mytable').filter(Filter().And(Filter().Or(Filter().gt('a', 2), Filter().lt('b', 2)), Filter().eq('b', 2)))
self.assertEqual(query, correct)
return
def test_simple_feature_collection_with_where(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where a > 2 ')
query = sql2gee._feature_collection
correct = FeatureCollection('ft:mytable').filter(Filter().gt('a', 2))
self.assertEqual(query, correct)
return
def test_feature_collection_simple_with_where_in_string(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where a in ("a", "b") ')
correct = FeatureCollection('ft:mytable').filter(Filter().inList('a', ['a', 'b']))
query = sql2gee._feature_collection
self.assertEqual(query, correct)
return
def tile_feature_collection(feature_collection: ee.FeatureCollection):
return ee.FeatureCollection(
feature_collection.iterate(
lambda feature, acc: ee.FeatureCollection(acc).merge(
tile_geometry(ee.Feature(feature).geometry())),
ee.FeatureCollection([])))