def _joinFilteredETI(self, et, i):
time_filter = EEFilter.equals(leftField="system:time_start",
rightField="system:time_start")
join = ee.Join.inner()
joinCollETI = EEImageCollection(join.apply(et, i, time_filter))
return joinCollETI.map(lambda element: EEImage.cat(
element.get('primary'), element.get('secondary'))).sort(
'system:time_start')
def execute(self):
# utils
m = MapUtil()
s = StatUtil()
# instantiate filters
t_filter = Filter(
DateRange(
self.filters["temporal_extent"]["startdate"],
self.filters["temporal_extent"]["enddate"]
)
)
g_filter = Geometry(
self.filters["spatial_extent"]
)
# instantiate input collections
coll_aeti = GEEUtil(self.__aeti.id)
coll_npp = GEEUtil(self.__npp.id)
# get rid of anomalies closed to no data values -9999
# lambda to filter: pixel >=0 over the NPP collection
coll_npp.filterValues(gte=0)
# reduce input collections with filters
# AETI
coll_aeti.filterDateRange(filter=t_filter)
coll_aeti.filterGeometry(filter=g_filter)
m_aeti = Map(
name="{}-{}".format(coll_aeti.collection, "filtered"),
rel=coll_aeti.collection,
url=coll_aeti.mapUrl(reduced=True)
)
m.add_map(m_aeti)
s_aeti = Stat(
name="{}-{}".format(coll_aeti.collection, "filtered"),
rel=coll_aeti.collection,
stat=coll_aeti.getStat(reduced=True))
s.add_stat(s_aeti)
# NPP
coll_npp.filterDateRange(filter=t_filter)
coll_npp.filterGeometry(filter=g_filter)
m_npp = Map(
name="{}-{}".format(coll_npp.collection, "filtered"),
rel=coll_npp.collection,
url=coll_npp.mapUrl(reduced=True))
m.add_map(m_npp)
s_npp = Stat(
name="{}-{}".format(coll_npp.collection, "filtered"),
rel=coll_npp.collection,
stat=coll_npp.getStat(reduced=True))
s.add_stat(s_npp)
# AGBP
coll_npp2agbp = coll_npp.reduced.map(
lambda image: image.multiply(0.01444).addBands(
image.metadata("n_days_extent")
)
)
coll_agbp_dk = coll_npp2agbp.map(
lambda image: image.select("b1").multiply(
image.select("n_days_extent")
)
).sum()
# ETA
coll_eta = coll_aeti.reduced.map(
lambda image: image.addBands(image.metadata("n_days_extent"))
)
coll_eta_dk = coll_eta.map(
lambda image: image.select("b1").multiply(
image.select("n_days_extent")
)
).sum()
# WPBM
wpbm = coll_agbp_dk.divide(coll_eta_dk)
# print("wpbm={}".format(wpbm.getInfo()))
m_wpbm = Map(
name="{}-{}".format("WPBM", "calculated"),
rel="WPBM",
url=createInstanceUrl(wpbm))
m.add_map(m_wpbm)
s_wpbm = Stat(
name="{}-{}".format("WPBM", "calculated"),
rel="WPBM",
stat=createImageStat(img_inst=wpbm, region=g_filter, band="b1"))
s.add_stat(s_wpbm)
# print("stats={}".format(createImageStat(img_inst=wpbm, region=g_filter, band="b1")))
self.__outputs["maps"] = m.maps
self.__outputs["stats"] = s.stats
return {"outputs": self.outputs, "errors": self.errors}
def test_feature_collection_simple_with_where_is_not(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where a is not NULL ')
correct = FeatureCollection('ft:mytable').filter(Filter().neq('a', 'null'))
query = sql2gee._feature_collection
self.assertEqual(query, correct)
return
def test_feature_collection_simple_with_where_in_string(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where a in ("a", "b") ')
correct = FeatureCollection('ft:mytable').filter(Filter().inList('a', ['a', 'b']))
query = sql2gee._feature_collection
self.assertEqual(query, correct)
return
def test_feature_collection_simple_with_where_string(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where a = "a" ')
correct = FeatureCollection('ft:mytable').filter(Filter().eq('a', "a"))
query = sql2gee._feature_collection
self.assertEqual(query, correct)
return
def test_feature_collection_simple_with_where_gt_and_like(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where b > 2 and a like "%2%"')
correct = FeatureCollection('ft:mytable').filter(Filter().And(Filter().gt('b', 2), Filter().stringContains('a', '2')))
query = sql2gee._feature_collection
self.assertEqual(query, correct)
return
def test_feature_collection_simple_with_where_not_like(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where a not like "%2%" ')
correct = FeatureCollection('ft:mytable').filter(Filter().stringContains('a', '2').Not())
query = sql2gee._feature_collection
self.assertEqual(query, correct)
return
def test_feature_collection_simple_with_where_like_endsWith(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where a like "%2" ')
correct = FeatureCollection('ft:mytable').filter(Filter().stringEndsWith('a', '2'))
query = sql2gee._feature_collection
self.assertEqual(query, correct)
return
def test_feature_collection_simple_with_where_not_float(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where not a > 2.2 ')
correct = FeatureCollection('ft:mytable').filter(Filter().gt('a', 2.2).Not())
query = sql2gee._feature_collection
self.assertEqual(query, correct)
return
def test_feature_collection_simple_with_where_several_conditions(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where (a > 2 or b < 2) and b = 2 ')
query = sql2gee._feature_collection
correct = FeatureCollection('ft:mytable').filter(Filter().And(Filter().Or(Filter().gt('a', 2), Filter().lt('b', 2)), Filter().eq('b', 2)))
self.assertEqual(query, correct)
return
def test_simple_feature_collection_with_where(self):
sql2gee = SQL2GEE('select * from "ft:mytable" where a > 2 ')
query = sql2gee._feature_collection
correct = FeatureCollection('ft:mytable').filter(Filter().gt('a', 2))
self.assertEqual(query, correct)
return
def test_where_with_string(self):
sql2gee = SQL2GEE('select * from mytable where a = "pepe"')
where = sql2gee.where
correct = Filter().eq('a', 'pepe')
self.assertEqual(where, correct)
return
def test_where_with_or_and_and(self):
sql2gee = SQL2GEE('select * from mytable where a > 2 and c = 2 or x <= 2')
where = sql2gee.where
correct = Filter().Or(Filter().And(Filter().gt('a', 2), Filter().eq('c', 2)), Filter().lte('x', 2))
self.assertEqual(where, correct)
return
def test_where_with_or(self):
sql2gee = SQL2GEE('select * from mytable where a > 2 or c = 2')
where = sql2gee.where
correct = Filter().Or(Filter().gt('a', 2), Filter().eq('c', 2))
self.assertEqual(where, correct)
def test_where_simple(self):
sql2gee = SQL2GEE('select * from mytable where a > 2')
where = sql2gee.where
correct = Filter().gt('a', 2)
self.assertEqual(where, correct)
return
def process_seasonal(self):
"""Calculate Seasonal NBWP image.
"""
self.logger.debug(
"Config dictionary =====> {0}".format(
json.dumps(self.config)
)
)
self._tasks = {}
# Filtered collection
self.logger.debug(
"temporal_filter GEE object value is ======> {0}".format(
self.filter["temporal_filter"]
)
)
# AGBP collection
collAGBP = self.coll["collection"].sort(
'system:time_start', True
)
# nodatavalue -9999: consider only gte 0
if self.config["ndvalue"] and "-9999" in self.config["ndvalue"]:
collAGBP = collAGBP.map(
lambda image: image.mask(
image.select('b1').gte(0)
)
)
else:
pass
collTFiltered = self.coll_t_y.filterDate(
self.filter["temporal_filter"]['start'],
self.filter["temporal_filter"]['end']
).sort('system:time_start', True)
# lambda to filter: pixel >=0 over the AGBP collection
AGBPf = collAGBP.map(
lambda image: image.updateMask(
image.gte(0)
)
)
# same annual plus season, be careful of gee number type
AGBPs_Im = AGBPf.filter(
EEFilter.eq('season', float(self.season))
).filterDate(
self.filter["temporal_filter"]['start'],
self.filter["temporal_filter"]['end']
)
# first element
first_agbpsim = AGBPs_Im.first()
# properties
first_agbpsim_info = first_agbpsim.getInfo()
bands = first_agbpsim_info["bands"][0]
dimensions = (bands["dimensions"][0], bands["dimensions"][1],)
seasonal_properties = first_agbpsim_info["properties"]
# Set an instance of phenology collection
phen = Phenology(
# static configuration from file
phe_coll="projects/fao-wapor/L2/L2_PHE_S",
year=self.year,
season=int(self.season)
)
# get phes_s image
phes_s = phen.PHEsos_img
# get phes_e image
phes_e = phen.PHEeos_img
# get phestart image
phestart = phen.PHEstart
# get pheend image
pheend = phen.PHEend
# get season_s image
season_s = phen.season_s
# get season_e image
season_e = phen.season_e
# calculate lenght of season in days
season_days = season_e.subtract(season_s).divide(86400000)
# adds band with own date to T Collection
TColl_date = collTFiltered.map(
lambda image: image.addBands(
image.metadata('system:time_start')
)
)
# filter T between PHEs and PHEe dates
T_season = TColl_date.map(
lambda image, seas=season_s, seae=season_e: image.updateMask(
image.select(['system:time_start']).gte(seas).And(
image.select(['system:time_start']).lte(seae)
)
)
)
# get total ETa in m3/ha = (ETa_mm*10000)/1000)
T_season_m3 = T_season.sum().multiply(10)
# calculate NBWP
NBWPSeason = first_agbpsim.divide(T_season_m3)
# multiplier
NBWP_seasonal = NBWPSeason.multiply(1000)
self.logger.debug(
"NBWP_seasonal info is =====> \n{0}".format(
NBWP_seasonal.getInfo()
)
)
NBWP_seasonal_int = NBWP_seasonal.select("b1").unmask(
-9999
).int32()
self.logger.debug(
"Config dictionary =====> {0}".format(
json.dumps(self.config)
)
)
assetid = self.config["assetid"]
asset_name = os.path.basename(assetid)
# seasonal_props for export
seasonal_props = self._setExportProperties(
self.year, asset_name, **seasonal_properties
)
NBWP_seasonal_props = ee.Image.setMulti(
NBWP_seasonal_int, seasonal_props
)
properties = NBWP_seasonal_props.getInfo()
self.logger.debug(
"New properties are =====>\n{0}".format(
json.dumps(properties)
)
)
pyramid_policy = json.dumps({"{0}".format(
properties["bands"][0]["id"]
): "mode"})
self.logger.debug(
"PyramidingPolicy is =====>\n{0}".format(
pyramid_policy
)
)
# check if the asset already exists and eventually delete it
if ee.data.getInfo(assetid):
try:
ee.data.deleteAsset(assetid)
except EEException as eee:
self.logger.debug(
"Trying to delete an assetId {0} \
which doesn't exist.".format(assetid)
)
raise
# launch the task and return taskid
try:
task = ee.batch.Export.image.toAsset(
image=NBWP_seasonal_props,
description=asset_name,
assetId=assetid,
crs=bands["crs"],
dimensions="{0}x{1}".format(
dimensions[0],
dimensions[1]
),
maxPixels=dimensions[0] * dimensions[1],
crsTransform=str(bands["crs_transform"]),
pyramidingPolicy=pyramid_policy
)
task.start()
self._tasks.update(
{
"{0}".format(assetid): {
"taskid": task.id
}
}
)
return dict(
tasks=self._tasks,
outputs=self.outputs,
errors=self.errors
)
except (EEException, AttributeError) as e:
self.logger.debug(
"Task export definition has failed with =====>\
\n{0}".format(e)
)
raise