def regenerate_overlay(overlay): if overlay is None: return eeservice.initEarthEngineService() # we need earth engine now. if not eeservice.initEarthEngineService(): # we need earth engine now. logging.info(initstr) logging.error('Could not contact Google Earth Engine to fetch latest area overlays') return map_id = eeservice.getLandsatImageById(overlay.image_collection, overlay.image_id, overlay.algorithm) overlay.map_id = map_id['mapid'] overlay.token = map_id['token'] overlay.put() return overlay
def create_drive_service(): if not eeservice.initEarthEngineService(): logging.error('Sorry, Server Credentials Error') credentials = eeservice.EarthEngineService.credentials http = credentials.authorize(httplib2.Http()) #return build(serviceName='drive', version='v3', http=http, credentials=credentials) #raised ValueError('Arguments http and credentials are mutually exclusive.') return build(serviceName='drive', version='v3', http=http)
def gladalerts_footprint_fc(): """ GLAD Alerts footprint. Geographic range of glad alerts in 2016 @returns: an ee.FeatureCollection of the GLAD ALERT footprint- 2016. @FIXME - Currently the returned footprint includes all of Peru, not just 'humid tropical Peru' """ if not eeservice.initEarthEngineService(): logging.error('Sorry, Server Credentials Error') return False #outline is a polygon surrounding kalimantan and no other part of Indonesia. outline_fc = ee.FeatureCollection( ee.Feature({ "type": "Polygon", "coordinates": [[[120.47607421874999, 4.23685605976896], [109.40185546874999, 4.784468966579375], [108.2373046875, -2.5479878714713835], [116.49902343749999, -4.8282597468669755], [120.47607421874999, 4.23685605976896]]] })) countries = ee.FeatureCollection( 'ft:1tdSwUL7MVpOauSgRzqVTOwdfy17KDbw-1d9omPw') congo = countries.filterMetadata('Country', 'equals', 'Congo') peru = countries.filterMetadata('Country', 'equals', 'Peru') indonesia = countries.filterMetadata('Country', 'equals', 'Indonesia') kalimantan = indonesia.geometry().intersection(outline_fc) footprint = congo.merge(peru).merge(kalimantan) return footprint
def gladalerts_footprint_fc(): """ GLAD Alerts footprint. Geographic range of glad alerts in 2016 @returns: an ee.FeatureCollection of the GLAD ALERT footprint- 2016. @FIXME - Currently the returned footprint includes all of Peru, not just 'humid tropical Peru' """ if not eeservice.initEarthEngineService(): logging.error('Sorry, Server Credentials Error') return False #outline is a polygon surrounding kalimantan and no other part of Indonesia. outline_fc = ee.FeatureCollection(ee.Feature({ "type": "Polygon", "coordinates": [[ [120.47607421874999, 4.23685605976896], [109.40185546874999, 4.784468966579375], [108.2373046875, -2.5479878714713835], [116.49902343749999, -4.8282597468669755], [120.47607421874999, 4.23685605976896] ]] })) countries = ee.FeatureCollection('ft:1tdSwUL7MVpOauSgRzqVTOwdfy17KDbw-1d9omPw') congo = countries.filterMetadata('Country', 'equals', 'Congo') peru = countries.filterMetadata('Country', 'equals', 'Peru') indonesia = countries.filterMetadata('Country', 'equals', 'Indonesia') kalimantan = indonesia.geometry().intersection(outline_fc) footprint = congo.merge(peru).merge(kalimantan) return footprint
def fetch_latest_overlay(area): """ :param area: :return: Latest LANDSAT image overlay for area. Creates new overlay if none already exists. """ if not area: return if not eeservice.initEarthEngineService(): # we need earth engine now. logging.error('Could not contact Google Earth Engine to fetch latest area overlays') return if area.get_boundary_hull_fc() is None: logging.error('Could not create overlay. Area %s does not have a valid location or boundary' % ( area.name)) return map_id = eeservice.getLandsatOverlay(area.get_boundary_hull_fc(), 'l8', 'rgb', 0, []) overlays = models.Overlay.query(models.Overlay.map_id == map_id['mapid']).fetch() if len(overlays) == 0: overlay = models.Overlay(image_id=map_id['id'], map_id=map_id['mapid'], token=map_id['token'], image_collection=map_id['collection'], algorithm='rgb') overlay.put() overlays = [overlay] return overlays
def create_table_service(): if not eeservice.initEarthEngineService(): logging.error('Sorry, Server Credentials Error') credentials = eeservice.EarthEngineService.credentials http = credentials.authorize(httplib2.Http()) return build(serviceName='fusiontables', version='v2', http=http, developerKey=settings.MY_LOCAL_SERVICE_ACCOUNT)
def find_regions(area_fc): """ @param: area_fc an ee.Geometry representing an area of interest defined by a user @returns: a list of regions intersecting with the provided area """ if not eeservice.initEarthEngineService(): logging.error('Sorry, Server Credentials Error') return False area_geom = area_fc.geometry().convexHull(100) regions = [] if _is_in_congo(area_geom): regions.append('congo') if _is_in_peru(area_geom): regions.append('peru') if _is_in_indonesia(area_geom): regions.append('indonesia') return regions
ft_docid = '' try: ft_docid = new_area['properties']['fusion_table']['ft_docid'].encode( 'utf-8') # allow non-english area names. except KeyError, e: logging.warning('Seeded area does not have a fusion table id.') existing_area = cache.get_area(area_name) if existing_area: remove_old_seeded_area(existing_area) logging.info('Deleted old seed data') # check EE service if not eeservice.initEarthEngineService(): # we need earth engine now. logging.error( 'Failed to seed data because Google Earth Engine not available right now. Please try again later' ) return False, None # pre-init variables to create area in case they don't appear in the geojson object maxlatlon = ndb.GeoPt(0, 0) minlatlon = ndb.GeoPt(0, 0) area_location = ndb.GeoPt(0, 0) boundary_hull = None boundary_geojsonstr = None # no boundary defined yet. # boundary_type = new_area['properties']['boundary_type'] boundary_feature = None # client may optionally pass boundary on a Feature named 'boundary' total_area = 0 # coords = []
return False, None ft_docid = '' try: ft_docid = new_area['properties']['fusion_table']['ft_docid'].encode('utf-8') # allow non-english area names. except KeyError, e: logging.warning('Seeded area does not have a fusion table id.') existing_area = cache.get_area(area_name) if existing_area: remove_old_seeded_area(existing_area) logging.info('Deleted old seed data') # check EE service if not eeservice.initEarthEngineService(): # we need earth engine now. logging.error('Failed to seed data because Google Earth Engine not available right now. Please try again later') return False, None # pre-init variables to create area in case they don't appear in the geojson object maxlatlon = ndb.GeoPt(0, 0) minlatlon = ndb.GeoPt(0, 0) area_location = ndb.GeoPt(0, 0) boundary_hull = None boundary_geojsonstr = None # no boundary defined yet. # boundary_type = new_area['properties']['boundary_type'] boundary_feature = None # client may optionally pass boundary on a Feature named 'boundary' total_area = 0 # coords = [] # ft_docid = None # center_pt = []
def alerts2Clusters(area, create_task=True): """ alerts2Clusters() calls Earth Engine to process the alerts to a FeatureCollection It Clusters alert points into polygons. Based on an answer from Noel Gorelich, GoogleGroups April 27 2016 https://groups.google.com/d/msg/google-earth-engine-developers/3Oq1t9dBUqE/ft50BYTxDQAJ Updated EarthEngine Code at https://code.earthengine.google.com/046c8e9074f9a9f2f4442dd53ce7ef94 @param area: an Area of Interest - reads the last raw alerts fusion table ID. eps = 600 (eps): the radius to look for neighbours. """ eps = 600 #(eps): the radius to look for neighbours. if not eeservice.initEarthEngineService(): logging.error('Sorry, Server Credentials Error') return "Could not connect to Earth Engine" ft = area.last_alerts_raw_ft raw_alerts_fc = ee.FeatureCollection("ft:" + ft) num_alerts = raw_alerts_fc.size().getInfo( ) # rows in ft = alerts for all dates. date_count = raw_alerts_fc.aggregate_count_distinct('date').getInfo() distinct_dates = raw_alerts_fc.distinct('date').sort( 'date', False) #false orders so first date is latest. latest_date_iso = distinct_dates.first().get("date").getInfo() latest_date = ee.Date(latest_date_iso).getInfo() #print 'latest_date_iso', latest_date_iso #print 'latest_date', latest_date alerts_fc = raw_alerts_fc.filterMetadata("date", "equals", latest_date_iso) latest_alerts = alerts_fc.size().getInfo( ) #number of alerts with latest date img = ee.Image(0).byte().paint(alerts_fc, 1) dist = img.distance(ee.Kernel.manhattan(eps * 6, "meters"), True) cluster_img = dist.lt(eps * 1) clusters = img.addBands(cluster_img).updateMask( cluster_img).reduceToVectors( reducer=ee.Reducer.first(), geometry=alerts_fc, geometryType='polygon', #or bb scale=eps, crs="EPSG:3857", bestEffort=True) buffer = eps * 0.1 clusters = clusters.map(lambda f: f.buffer(buffer, buffer).set( {'AreaInHa': f.geometry().area(100).divide(100 * 100)})) #select removes unnecessary properties. clusters = clusters.select(['points', 'AlertsInCluster', 'AreaInHa']) #hulls = clusters.map(lambda f: f.convexHull(10)) # not using convex Hulls. """ Join Cluster Feature Collection with List of Points """ #Define a spatial filter, with distance < 10. distFilter = ee.Filter.withinDistance(distance=10, leftField='.geo', rightField='.geo', maxError=10) distSaveAll = ee.Join.saveAll(matchesKey='points', measureKey='distance') clustersWithPoints = distSaveAll.apply(clusters, alerts_fc, distFilter) #Add a Count of points in each cluster clustersWithPoints = clustersWithPoints.map(lambda f: f.set({ 'AlertsInCluster': ee.List(ee.Feature(f).get('points')).length(), 'AlertsDate': ee.Feature(ee.List(ee.Feature(f).get('points')).get(0)).get('date'), # Add a pink border and no fill styling for geojson.io "stroke": "#ff6fb7", "stroke-width": 1, "fill-opacity": "0.0", })) most_populous_cluster = clustersWithPoints.reduceColumns( reducer=ee.Reducer.max(), selectors=['AlertsInCluster']) since_date = area.last_glad_alerts_date.strftime('%Y-%m-%d') today_dt = datetime.datetime.today() today_str = today_dt.strftime('%Y-%m-%d') #today_str = '2016-05-18' num_clusters = clusters.size().getInfo() foldername = area.name filename = area.name + "_GLADClusters_" if since_date: filename += "Since_" + since_date clusterProperties_dict = { 'name': 'gladclusters', 'filename': filename, 'epsilon': eps, 'area': area.name, 'ft': 'https://www.google.com/fusiontables/DataSource?docid=' + ft, 'since_date': since_date, 'clustered_date': today_str, 'alerts_date': latest_date_iso, #iso format 'num_clusters': num_clusters, 'distinct_dates': date_count, 'num_alerts_alldates': num_alerts, 'most_alerts_in_cluster': most_populous_cluster.get('max').getInfo() # More properties are added below } clustersWithPoints = clustersWithPoints.map(lambda f: f.set( 'points', ee.Algorithms.GeometryConstructors.MultiPoint( ee.List(ee.Feature(f).get('points')).map(lambda p: ee.Feature(p). geometry())))) list = ee.List([]) image_collection = ee.ImageCollection('LANDSAT/LC8_L1T_TOA').filterDate( '2016-04-01', '2016-11-09') clustersWithPoints = clustersWithPoints.map( lambda f: f.set({'image_collection' : image_collection.get('link'), 'LANDSAT_Latest': image_collection\ .filterBounds(ee.Feature(f).geometry()).limit(2, 'system:time_start', False)\ .iterate(addToList, list) })) ''' clustersWithPoints = clustersWithPoints.map( lambda f:f.set({'LANDSAT_Latest': image_collection .filterBounds(ee.Feature(f).geometry()).limit(2, 'system:time_start', False) .iterate( lambda image, list: (ee.List(list).add(list, ee.Dictionary({ 'image_id': image.id(), 'collection': image_collection.get('link'), 'path': image.get('WRS_PATH'), 'row': image.get('WRS_ROW'), 'captured': image.get('system:time_start') }))) , list) })) ''' clusterProperties_dict['LANDSAT_Latest'] = clustersWithPoints.get( 'LANDSAT_Latest') clusterProperties_dict['image_collection'] = clustersWithPoints.get( 'image_collection') c = clustersWithPoints.getInfo() print c dt = datetime.datetime.now() taskName = dt.strftime(filename + "_Exported_%a%Y%b%d_%H%M") task = ee.batch.Export.table.toDrive(clustersWithPoints, description=taskName, folder=foldername, fileNamePrefix=filename, fileFormat='geoJSON') ''' ctask = ee.batch.Export.table.toCloudStorage collection=clustersWithPoints, description=taskName, bucket='bfw-ee-cluster-tables', fileNamePrefix=filename, fileFormat='geoJSON' ) ''' task.start() #ctask.start() if create_task: #wait for export to finish then create a task. deferred.defer(check_export_status, task.id, clusterProperties_dict, _countdown=10, _queue="export-check-queue") logging.info("Started task to export GLAD Cluster") return "Exporting alerts to GLAD Cluster"
def alerts2Clusters(area, create_task=True): """ alerts2Clusters() calls Earth Engine to process the alerts to a FeatureCollection It Clusters alert points into polygons. Based on an answer from Noel Gorelich, GoogleGroups April 27 2016 https://groups.google.com/d/msg/google-earth-engine-developers/3Oq1t9dBUqE/ft50BYTxDQAJ Updated EarthEngine Code at https://code.earthengine.google.com/046c8e9074f9a9f2f4442dd53ce7ef94 @param area: an Area of Interest - reads the last raw alerts fusion table ID. eps = 600 (eps): the radius to look for neighbours. """ eps = 600 #(eps): the radius to look for neighbours. if not eeservice.initEarthEngineService(): logging.error('Sorry, Server Credentials Error') return "Could not connect to Earth Engine" ft = area.last_alerts_raw_ft raw_alerts_fc = ee.FeatureCollection("ft:" + ft) num_alerts = raw_alerts_fc.size().getInfo() # rows in ft = alerts for all dates. date_count = raw_alerts_fc.aggregate_count_distinct('date').getInfo() distinct_dates = raw_alerts_fc.distinct('date').sort('date', False) #false orders so first date is latest. latest_date_iso = distinct_dates.first().get("date").getInfo() latest_date = ee.Date(latest_date_iso).getInfo() #print 'latest_date_iso', latest_date_iso #print 'latest_date', latest_date alerts_fc = raw_alerts_fc.filterMetadata("date", "equals", latest_date_iso) latest_alerts = alerts_fc.size().getInfo() #number of alerts with latest date img = ee.Image(0).byte().paint(alerts_fc, 1) dist = img.distance(ee.Kernel.manhattan(eps * 6, "meters"), True) cluster_img = dist.lt(eps * 1) clusters = img.addBands(cluster_img).updateMask(cluster_img).reduceToVectors( reducer=ee.Reducer.first(), geometry=alerts_fc, geometryType='polygon', #or bb scale=eps, crs = "EPSG:3857", bestEffort = True ) buffer = eps * 0.1 clusters = clusters.map(lambda f: f.buffer(buffer, buffer).set({'AreaInHa': f.geometry().area(100).divide(100 * 100)})) #select removes unnecessary properties. clusters = clusters.select(['points', 'AlertsInCluster', 'AreaInHa']) #hulls = clusters.map(lambda f: f.convexHull(10)) # not using convex Hulls. """ Join Cluster Feature Collection with List of Points """ #Define a spatial filter, with distance < 10. distFilter = ee.Filter.withinDistance( distance = 10, leftField = '.geo', rightField = '.geo', maxError = 10 ) distSaveAll = ee.Join.saveAll(matchesKey = 'points', measureKey ='distance') clustersWithPoints = distSaveAll.apply(clusters, alerts_fc, distFilter) #Add a Count of points in each cluster clustersWithPoints = clustersWithPoints.map(lambda f: f.set({ 'AlertsInCluster': ee.List(ee.Feature(f).get('points')).length(), 'AlertsDate': ee.Feature(ee.List(ee.Feature(f).get('points')).get(0)).get('date'), # Add a pink border and no fill styling for geojson.io "stroke": "#ff6fb7", "stroke-width": 1, "fill-opacity": "0.0", })) most_populous_cluster = clustersWithPoints.reduceColumns( reducer=ee.Reducer.max(), selectors=['AlertsInCluster'] ) since_date = area.last_glad_alerts_date.strftime('%Y-%m-%d') today_dt = datetime.datetime.today() today_str = today_dt.strftime('%Y-%m-%d') #today_str = '2016-05-18' num_clusters = clusters.size().getInfo() foldername = area.name filename = area.name + "_GLADClusters_" if since_date: filename += "Since_" + since_date clusterProperties_dict = { 'name': 'gladclusters', 'filename': filename, 'epsilon': eps, 'area': area.name, 'ft': 'https://www.google.com/fusiontables/DataSource?docid=' + ft, 'since_date': since_date, 'clustered_date': today_str, 'alerts_date': latest_date_iso, #iso format 'num_clusters': num_clusters, 'distinct_dates': date_count, 'num_alerts_alldates': num_alerts, 'most_alerts_in_cluster': most_populous_cluster.get('max').getInfo() # More properties are added below } clustersWithPoints = clustersWithPoints.map( lambda f:f.set('points', ee.Algorithms.GeometryConstructors.MultiPoint( ee.List(ee.Feature(f).get('points')).map(lambda p : ee.Feature(p).geometry())))) list = ee.List([]) image_collection = ee.ImageCollection('LANDSAT/LC8_L1T_TOA').filterDate('2016-04-01', '2016-11-09') clustersWithPoints = clustersWithPoints.map( lambda f: f.set({'image_collection' : image_collection.get('link'), 'LANDSAT_Latest': image_collection\ .filterBounds(ee.Feature(f).geometry()).limit(2, 'system:time_start', False)\ .iterate(addToList, list) })) ''' clustersWithPoints = clustersWithPoints.map( lambda f:f.set({'LANDSAT_Latest': image_collection .filterBounds(ee.Feature(f).geometry()).limit(2, 'system:time_start', False) .iterate( lambda image, list: (ee.List(list).add(list, ee.Dictionary({ 'image_id': image.id(), 'collection': image_collection.get('link'), 'path': image.get('WRS_PATH'), 'row': image.get('WRS_ROW'), 'captured': image.get('system:time_start') }))) , list) })) ''' clusterProperties_dict['LANDSAT_Latest'] = clustersWithPoints.get('LANDSAT_Latest') clusterProperties_dict['image_collection'] = clustersWithPoints.get('image_collection') c = clustersWithPoints.getInfo() print c dt = datetime.datetime.now() taskName = dt.strftime(filename + "_Exported_%a%Y%b%d_%H%M") task = ee.batch.Export.table.toDrive(clustersWithPoints, description=taskName, folder=foldername, fileNamePrefix=filename, fileFormat='geoJSON') ''' ctask = ee.batch.Export.table.toCloudStorage collection=clustersWithPoints, description=taskName, bucket='bfw-ee-cluster-tables', fileNamePrefix=filename, fileFormat='geoJSON' ) ''' task.start() #ctask.start() if create_task: #wait for export to finish then create a task. deferred.defer(check_export_status, task.id, clusterProperties_dict, _countdown=10, _queue="export-check-queue") logging.info("Started task to export GLAD Cluster") return "Exporting alerts to GLAD Cluster"