def regenerate_overlay(overlay):
if overlay is None:
return
eeservice.initEarthEngineService() # we need earth engine now.
if not eeservice.initEarthEngineService(): # we need earth engine now. logging.info(initstr)
logging.error('Could not contact Google Earth Engine to fetch latest area overlays')
return
map_id = eeservice.getLandsatImageById(overlay.image_collection, overlay.image_id, overlay.algorithm)
overlay.map_id = map_id['mapid']
overlay.token = map_id['token']
overlay.put()
return overlay
def create_drive_service():
if not eeservice.initEarthEngineService():
logging.error('Sorry, Server Credentials Error')
credentials = eeservice.EarthEngineService.credentials
http = credentials.authorize(httplib2.Http())
#return build(serviceName='drive', version='v3', http=http, credentials=credentials) #raised ValueError('Arguments http and credentials are mutually exclusive.')
return build(serviceName='drive', version='v3', http=http)
def create_drive_service():
if not eeservice.initEarthEngineService():
logging.error('Sorry, Server Credentials Error')
credentials = eeservice.EarthEngineService.credentials
http = credentials.authorize(httplib2.Http())
#return build(serviceName='drive', version='v3', http=http, credentials=credentials) #raised ValueError('Arguments http and credentials are mutually exclusive.')
return build(serviceName='drive', version='v3', http=http)
def gladalerts_footprint_fc():
"""
GLAD Alerts footprint.
Geographic range of glad alerts in 2016
@returns: an ee.FeatureCollection of the GLAD ALERT footprint- 2016.
@FIXME - Currently the returned footprint includes all of Peru,
not just 'humid tropical Peru'
"""
if not eeservice.initEarthEngineService():
logging.error('Sorry, Server Credentials Error')
return False
#outline is a polygon surrounding kalimantan and no other part of Indonesia.
outline_fc = ee.FeatureCollection(
ee.Feature({
"type":
"Polygon",
"coordinates": [[[120.47607421874999, 4.23685605976896],
[109.40185546874999, 4.784468966579375],
[108.2373046875, -2.5479878714713835],
[116.49902343749999, -4.8282597468669755],
[120.47607421874999, 4.23685605976896]]]
}))
countries = ee.FeatureCollection(
'ft:1tdSwUL7MVpOauSgRzqVTOwdfy17KDbw-1d9omPw')
congo = countries.filterMetadata('Country', 'equals', 'Congo')
peru = countries.filterMetadata('Country', 'equals', 'Peru')
indonesia = countries.filterMetadata('Country', 'equals', 'Indonesia')
kalimantan = indonesia.geometry().intersection(outline_fc)
footprint = congo.merge(peru).merge(kalimantan)
return footprint
def gladalerts_footprint_fc():
"""
GLAD Alerts footprint.
Geographic range of glad alerts in 2016
@returns: an ee.FeatureCollection of the GLAD ALERT footprint- 2016.
@FIXME - Currently the returned footprint includes all of Peru,
not just 'humid tropical Peru'
"""
if not eeservice.initEarthEngineService():
logging.error('Sorry, Server Credentials Error')
return False
#outline is a polygon surrounding kalimantan and no other part of Indonesia.
outline_fc = ee.FeatureCollection(ee.Feature({
"type": "Polygon",
"coordinates": [[
[120.47607421874999, 4.23685605976896],
[109.40185546874999, 4.784468966579375],
[108.2373046875, -2.5479878714713835],
[116.49902343749999, -4.8282597468669755],
[120.47607421874999, 4.23685605976896]
]]
}))
countries = ee.FeatureCollection('ft:1tdSwUL7MVpOauSgRzqVTOwdfy17KDbw-1d9omPw')
congo = countries.filterMetadata('Country', 'equals', 'Congo')
peru = countries.filterMetadata('Country', 'equals', 'Peru')
indonesia = countries.filterMetadata('Country', 'equals', 'Indonesia')
kalimantan = indonesia.geometry().intersection(outline_fc)
footprint = congo.merge(peru).merge(kalimantan)
return footprint
def fetch_latest_overlay(area):
"""
:param area:
:return: Latest LANDSAT image overlay for area. Creates new overlay if none already exists.
"""
if not area:
return
if not eeservice.initEarthEngineService(): # we need earth engine now.
logging.error('Could not contact Google Earth Engine to fetch latest area overlays')
return
if area.get_boundary_hull_fc() is None:
logging.error('Could not create overlay. Area %s does not have a valid location or boundary' % (
area.name))
return
map_id = eeservice.getLandsatOverlay(area.get_boundary_hull_fc(), 'l8', 'rgb', 0, [])
overlays = models.Overlay.query(models.Overlay.map_id == map_id['mapid']).fetch()
if len(overlays) == 0:
overlay = models.Overlay(image_id=map_id['id'],
map_id=map_id['mapid'],
token=map_id['token'],
image_collection=map_id['collection'],
algorithm='rgb')
overlay.put()
overlays = [overlay]
return overlays
def create_table_service():
if not eeservice.initEarthEngineService():
logging.error('Sorry, Server Credentials Error')
credentials = eeservice.EarthEngineService.credentials
http = credentials.authorize(httplib2.Http())
return build(serviceName='fusiontables',
version='v2',
http=http,
developerKey=settings.MY_LOCAL_SERVICE_ACCOUNT)
def find_regions(area_fc):
"""
@param: area_fc an ee.Geometry representing an area of interest defined by a user
@returns: a list of regions intersecting with the provided area
"""
if not eeservice.initEarthEngineService():
logging.error('Sorry, Server Credentials Error')
return False
area_geom = area_fc.geometry().convexHull(100)
regions = []
if _is_in_congo(area_geom):
regions.append('congo')
if _is_in_peru(area_geom):
regions.append('peru')
if _is_in_indonesia(area_geom):
regions.append('indonesia')
return regions
def find_regions(area_fc):
"""
@param: area_fc an ee.Geometry representing an area of interest defined by a user
@returns: a list of regions intersecting with the provided area
"""
if not eeservice.initEarthEngineService():
logging.error('Sorry, Server Credentials Error')
return False
area_geom = area_fc.geometry().convexHull(100)
regions = []
if _is_in_congo(area_geom):
regions.append('congo')
if _is_in_peru(area_geom):
regions.append('peru')
if _is_in_indonesia(area_geom):
regions.append('indonesia')
return regions
def create_table_service():
if not eeservice.initEarthEngineService():
logging.error('Sorry, Server Credentials Error')
credentials = eeservice.EarthEngineService.credentials
http = credentials.authorize(httplib2.Http())
return build(serviceName='fusiontables', version='v2', http=http, developerKey=settings.MY_LOCAL_SERVICE_ACCOUNT)
ft_docid = ''
try:
ft_docid = new_area['properties']['fusion_table']['ft_docid'].encode(
'utf-8') # allow non-english area names.
except KeyError, e:
logging.warning('Seeded area does not have a fusion table id.')
existing_area = cache.get_area(area_name)
if existing_area:
remove_old_seeded_area(existing_area)
logging.info('Deleted old seed data')
# check EE service
if not eeservice.initEarthEngineService(): # we need earth engine now.
logging.error(
'Failed to seed data because Google Earth Engine not available right now. Please try again later'
)
return False, None
# pre-init variables to create area in case they don't appear in the geojson object
maxlatlon = ndb.GeoPt(0, 0)
minlatlon = ndb.GeoPt(0, 0)
area_location = ndb.GeoPt(0, 0)
boundary_hull = None
boundary_geojsonstr = None # no boundary defined yet.
# boundary_type = new_area['properties']['boundary_type']
boundary_feature = None # client may optionally pass boundary on a Feature named 'boundary'
total_area = 0
# coords = []
return False, None
ft_docid = ''
try:
ft_docid = new_area['properties']['fusion_table']['ft_docid'].encode('utf-8') # allow non-english area names.
except KeyError, e:
logging.warning('Seeded area does not have a fusion table id.')
existing_area = cache.get_area(area_name)
if existing_area:
remove_old_seeded_area(existing_area)
logging.info('Deleted old seed data')
# check EE service
if not eeservice.initEarthEngineService(): # we need earth engine now.
logging.error('Failed to seed data because Google Earth Engine not available right now. Please try again later')
return False, None
# pre-init variables to create area in case they don't appear in the geojson object
maxlatlon = ndb.GeoPt(0, 0)
minlatlon = ndb.GeoPt(0, 0)
area_location = ndb.GeoPt(0, 0)
boundary_hull = None
boundary_geojsonstr = None # no boundary defined yet.
# boundary_type = new_area['properties']['boundary_type']
boundary_feature = None # client may optionally pass boundary on a Feature named 'boundary'
total_area = 0
# coords = []
# ft_docid = None
# center_pt = []
def alerts2Clusters(area, create_task=True):
"""
alerts2Clusters() calls Earth Engine to process the alerts to a FeatureCollection
It Clusters alert points into polygons.
Based on an answer from Noel Gorelich, GoogleGroups April 27 2016
https://groups.google.com/d/msg/google-earth-engine-developers/3Oq1t9dBUqE/ft50BYTxDQAJ
Updated EarthEngine Code at https://code.earthengine.google.com/046c8e9074f9a9f2f4442dd53ce7ef94
@param area: an Area of Interest - reads the last raw alerts fusion table ID.
eps = 600 (eps): the radius to look for neighbours.
"""
eps = 600 #(eps): the radius to look for neighbours.
if not eeservice.initEarthEngineService():
logging.error('Sorry, Server Credentials Error')
return "Could not connect to Earth Engine"
ft = area.last_alerts_raw_ft
raw_alerts_fc = ee.FeatureCollection("ft:" + ft)
num_alerts = raw_alerts_fc.size().getInfo(
) # rows in ft = alerts for all dates.
date_count = raw_alerts_fc.aggregate_count_distinct('date').getInfo()
distinct_dates = raw_alerts_fc.distinct('date').sort(
'date', False) #false orders so first date is latest.
latest_date_iso = distinct_dates.first().get("date").getInfo()
latest_date = ee.Date(latest_date_iso).getInfo()
#print 'latest_date_iso', latest_date_iso
#print 'latest_date', latest_date
alerts_fc = raw_alerts_fc.filterMetadata("date", "equals", latest_date_iso)
latest_alerts = alerts_fc.size().getInfo(
) #number of alerts with latest date
img = ee.Image(0).byte().paint(alerts_fc, 1)
dist = img.distance(ee.Kernel.manhattan(eps * 6, "meters"), True)
cluster_img = dist.lt(eps * 1)
clusters = img.addBands(cluster_img).updateMask(
cluster_img).reduceToVectors(
reducer=ee.Reducer.first(),
geometry=alerts_fc,
geometryType='polygon', #or bb
scale=eps,
crs="EPSG:3857",
bestEffort=True)
buffer = eps * 0.1
clusters = clusters.map(lambda f: f.buffer(buffer, buffer).set(
{'AreaInHa': f.geometry().area(100).divide(100 * 100)}))
#select removes unnecessary properties.
clusters = clusters.select(['points', 'AlertsInCluster', 'AreaInHa'])
#hulls = clusters.map(lambda f: f.convexHull(10)) # not using convex Hulls.
"""
Join Cluster Feature Collection with List of Points
"""
#Define a spatial filter, with distance < 10.
distFilter = ee.Filter.withinDistance(distance=10,
leftField='.geo',
rightField='.geo',
maxError=10)
distSaveAll = ee.Join.saveAll(matchesKey='points', measureKey='distance')
clustersWithPoints = distSaveAll.apply(clusters, alerts_fc, distFilter)
#Add a Count of points in each cluster
clustersWithPoints = clustersWithPoints.map(lambda f: f.set({
'AlertsInCluster':
ee.List(ee.Feature(f).get('points')).length(),
'AlertsDate':
ee.Feature(ee.List(ee.Feature(f).get('points')).get(0)).get('date'),
# Add a pink border and no fill styling for geojson.io
"stroke":
"#ff6fb7",
"stroke-width":
1,
"fill-opacity":
"0.0",
}))
most_populous_cluster = clustersWithPoints.reduceColumns(
reducer=ee.Reducer.max(), selectors=['AlertsInCluster'])
since_date = area.last_glad_alerts_date.strftime('%Y-%m-%d')
today_dt = datetime.datetime.today()
today_str = today_dt.strftime('%Y-%m-%d') #today_str = '2016-05-18'
num_clusters = clusters.size().getInfo()
foldername = area.name
filename = area.name + "_GLADClusters_"
if since_date:
filename += "Since_" + since_date
clusterProperties_dict = {
'name': 'gladclusters',
'filename': filename,
'epsilon': eps,
'area': area.name,
'ft': 'https://www.google.com/fusiontables/DataSource?docid=' + ft,
'since_date': since_date,
'clustered_date': today_str,
'alerts_date': latest_date_iso, #iso format
'num_clusters': num_clusters,
'distinct_dates': date_count,
'num_alerts_alldates': num_alerts,
'most_alerts_in_cluster': most_populous_cluster.get('max').getInfo()
# More properties are added below
}
clustersWithPoints = clustersWithPoints.map(lambda f: f.set(
'points',
ee.Algorithms.GeometryConstructors.MultiPoint(
ee.List(ee.Feature(f).get('points')).map(lambda p: ee.Feature(p).
geometry()))))
list = ee.List([])
image_collection = ee.ImageCollection('LANDSAT/LC8_L1T_TOA').filterDate(
'2016-04-01', '2016-11-09')
clustersWithPoints = clustersWithPoints.map(
lambda f: f.set({'image_collection' : image_collection.get('link'),
'LANDSAT_Latest': image_collection\
.filterBounds(ee.Feature(f).geometry()).limit(2, 'system:time_start', False)\
.iterate(addToList, list)
}))
'''
clustersWithPoints = clustersWithPoints.map(
lambda f:f.set({'LANDSAT_Latest': image_collection
.filterBounds(ee.Feature(f).geometry()).limit(2, 'system:time_start', False)
.iterate(
lambda image, list: (ee.List(list).add(list, ee.Dictionary({
'image_id': image.id(),
'collection': image_collection.get('link'),
'path': image.get('WRS_PATH'),
'row': image.get('WRS_ROW'),
'captured': image.get('system:time_start')
})))
, list)
}))
'''
clusterProperties_dict['LANDSAT_Latest'] = clustersWithPoints.get(
'LANDSAT_Latest')
clusterProperties_dict['image_collection'] = clustersWithPoints.get(
'image_collection')
c = clustersWithPoints.getInfo()
print c
dt = datetime.datetime.now()
taskName = dt.strftime(filename + "_Exported_%a%Y%b%d_%H%M")
task = ee.batch.Export.table.toDrive(clustersWithPoints,
description=taskName,
folder=foldername,
fileNamePrefix=filename,
fileFormat='geoJSON')
'''
ctask = ee.batch.Export.table.toCloudStorage
collection=clustersWithPoints,
description=taskName,
bucket='bfw-ee-cluster-tables',
fileNamePrefix=filename,
fileFormat='geoJSON'
)
'''
task.start()
#ctask.start()
if create_task:
#wait for export to finish then create a task.
deferred.defer(check_export_status,
task.id,
clusterProperties_dict,
_countdown=10,
_queue="export-check-queue")
logging.info("Started task to export GLAD Cluster")
return "Exporting alerts to GLAD Cluster"
def alerts2Clusters(area, create_task=True):
"""
alerts2Clusters() calls Earth Engine to process the alerts to a FeatureCollection
It Clusters alert points into polygons.
Based on an answer from Noel Gorelich, GoogleGroups April 27 2016
https://groups.google.com/d/msg/google-earth-engine-developers/3Oq1t9dBUqE/ft50BYTxDQAJ
Updated EarthEngine Code at https://code.earthengine.google.com/046c8e9074f9a9f2f4442dd53ce7ef94
@param area: an Area of Interest - reads the last raw alerts fusion table ID.
eps = 600 (eps): the radius to look for neighbours.
"""
eps = 600 #(eps): the radius to look for neighbours.
if not eeservice.initEarthEngineService():
logging.error('Sorry, Server Credentials Error')
return "Could not connect to Earth Engine"
ft = area.last_alerts_raw_ft
raw_alerts_fc = ee.FeatureCollection("ft:" + ft)
num_alerts = raw_alerts_fc.size().getInfo() # rows in ft = alerts for all dates.
date_count = raw_alerts_fc.aggregate_count_distinct('date').getInfo()
distinct_dates = raw_alerts_fc.distinct('date').sort('date', False) #false orders so first date is latest.
latest_date_iso = distinct_dates.first().get("date").getInfo()
latest_date = ee.Date(latest_date_iso).getInfo()
#print 'latest_date_iso', latest_date_iso
#print 'latest_date', latest_date
alerts_fc = raw_alerts_fc.filterMetadata("date", "equals", latest_date_iso)
latest_alerts = alerts_fc.size().getInfo() #number of alerts with latest date
img = ee.Image(0).byte().paint(alerts_fc, 1)
dist = img.distance(ee.Kernel.manhattan(eps * 6, "meters"), True)
cluster_img = dist.lt(eps * 1)
clusters = img.addBands(cluster_img).updateMask(cluster_img).reduceToVectors(
reducer=ee.Reducer.first(),
geometry=alerts_fc,
geometryType='polygon', #or bb
scale=eps,
crs = "EPSG:3857",
bestEffort = True
)
buffer = eps * 0.1
clusters = clusters.map(lambda f: f.buffer(buffer, buffer).set({'AreaInHa': f.geometry().area(100).divide(100 * 100)}))
#select removes unnecessary properties.
clusters = clusters.select(['points', 'AlertsInCluster', 'AreaInHa'])
#hulls = clusters.map(lambda f: f.convexHull(10)) # not using convex Hulls.
"""
Join Cluster Feature Collection with List of Points
"""
#Define a spatial filter, with distance < 10.
distFilter = ee.Filter.withinDistance(
distance = 10,
leftField = '.geo',
rightField = '.geo',
maxError = 10
)
distSaveAll = ee.Join.saveAll(matchesKey = 'points', measureKey ='distance')
clustersWithPoints = distSaveAll.apply(clusters, alerts_fc, distFilter)
#Add a Count of points in each cluster
clustersWithPoints = clustersWithPoints.map(lambda f: f.set({
'AlertsInCluster': ee.List(ee.Feature(f).get('points')).length(),
'AlertsDate': ee.Feature(ee.List(ee.Feature(f).get('points')).get(0)).get('date'),
# Add a pink border and no fill styling for geojson.io
"stroke": "#ff6fb7",
"stroke-width": 1,
"fill-opacity": "0.0",
}))
most_populous_cluster = clustersWithPoints.reduceColumns(
reducer=ee.Reducer.max(),
selectors=['AlertsInCluster']
)
since_date = area.last_glad_alerts_date.strftime('%Y-%m-%d')
today_dt = datetime.datetime.today()
today_str = today_dt.strftime('%Y-%m-%d') #today_str = '2016-05-18'
num_clusters = clusters.size().getInfo()
foldername = area.name
filename = area.name + "_GLADClusters_"
if since_date:
filename += "Since_" + since_date
clusterProperties_dict = {
'name': 'gladclusters',
'filename': filename,
'epsilon': eps,
'area': area.name,
'ft': 'https://www.google.com/fusiontables/DataSource?docid=' + ft,
'since_date': since_date,
'clustered_date': today_str,
'alerts_date': latest_date_iso, #iso format
'num_clusters': num_clusters,
'distinct_dates': date_count,
'num_alerts_alldates': num_alerts,
'most_alerts_in_cluster': most_populous_cluster.get('max').getInfo()
# More properties are added below
}
clustersWithPoints = clustersWithPoints.map(
lambda f:f.set('points', ee.Algorithms.GeometryConstructors.MultiPoint(
ee.List(ee.Feature(f).get('points')).map(lambda p : ee.Feature(p).geometry()))))
list = ee.List([])
image_collection = ee.ImageCollection('LANDSAT/LC8_L1T_TOA').filterDate('2016-04-01', '2016-11-09')
clustersWithPoints = clustersWithPoints.map(
lambda f: f.set({'image_collection' : image_collection.get('link'),
'LANDSAT_Latest': image_collection\
.filterBounds(ee.Feature(f).geometry()).limit(2, 'system:time_start', False)\
.iterate(addToList, list)
}))
'''
clustersWithPoints = clustersWithPoints.map(
lambda f:f.set({'LANDSAT_Latest': image_collection
.filterBounds(ee.Feature(f).geometry()).limit(2, 'system:time_start', False)
.iterate(
lambda image, list: (ee.List(list).add(list, ee.Dictionary({
'image_id': image.id(),
'collection': image_collection.get('link'),
'path': image.get('WRS_PATH'),
'row': image.get('WRS_ROW'),
'captured': image.get('system:time_start')
})))
, list)
}))
'''
clusterProperties_dict['LANDSAT_Latest'] = clustersWithPoints.get('LANDSAT_Latest')
clusterProperties_dict['image_collection'] = clustersWithPoints.get('image_collection')
c = clustersWithPoints.getInfo()
print c
dt = datetime.datetime.now()
taskName = dt.strftime(filename + "_Exported_%a%Y%b%d_%H%M")
task = ee.batch.Export.table.toDrive(clustersWithPoints, description=taskName,
folder=foldername, fileNamePrefix=filename, fileFormat='geoJSON')
'''
ctask = ee.batch.Export.table.toCloudStorage
collection=clustersWithPoints,
description=taskName,
bucket='bfw-ee-cluster-tables',
fileNamePrefix=filename,
fileFormat='geoJSON'
)
'''
task.start()
#ctask.start()
if create_task:
#wait for export to finish then create a task.
deferred.defer(check_export_status, task.id, clusterProperties_dict, _countdown=10, _queue="export-check-queue")
logging.info("Started task to export GLAD Cluster")
return "Exporting alerts to GLAD Cluster"