def run(self):
logger.info(
'Monitoring Earth Engine export. task_id: {0}, destination: {1}'.
format(self.task_id, self.destination))
ee.InitializeThread(self.credentials)
while self.running():
time.sleep(10)
status = Task(self.task_id).status()
state = status['state']
if state == Task.State.READY:
self._update_state(State.PENDING)
elif state == Task.State.RUNNING:
self._update_state(State.ACTIVE)
elif state == Task.State.COMPLETED:
self._update_state(State.COMPLETED)
self.resolve()
return
else:
error_message = status[
'error_message'] if state == Task.State.FAILED else 'Earth Engine export was canceled'
self._status = {
'state': State.FAILED,
'message': error_message
}
raise Exception(error_message if error_message else
'Earth Engine export failed')
def run(self):
ee.InitializeThread(self.credentials)
task_id = self._image_to_asset()
self._monitor = MonitorEarthEngineExportTask(self.credentials, task_id, self.description)
self.dependent(self._monitor) \
.submit() \
.then(self.resolve, self.reject)
def _smooth(self):
tasks = []
for primitive_type in self.primitive_types:
primitive_collection = ee.ImageCollection([
_primitive_asset_id(self.asset_path, year, primitive_type)
for year in range(self.start_year, self.end_year + 1)
])
smoothed_primitive_collection, rmse = smooth_primitive_collection(
primitive_collection)
for year in range(self.start_year, self.end_year + 1):
smoothed_primitive = ee.Image(
smoothed_primitive_collection.filterDate(
ee.Date.fromYMD(year, 1, 1),
ee.Date.fromYMD(year + 1, 1, 1)).first())
tasks.append(
self.dependent(
ImageToAsset(credentials=self.credentials,
image=smoothed_primitive,
region=self.aoi.bounds(),
description=None,
assetId=_smoothed_primitive_asset_id(
self.asset_path, year,
primitive_type),
scale=self.scale,
retries=5)))
ee.InitializeThread(self.credentials)
return Task.submit_all(tasks)
def run(self):
ee.InitializeThread(self.spec.credentials)
# Explicitly create folder, to prevent GEE race-condition
self.drive_folder = drive.create_folder(self.spec.credentials,
self.drive_folder_name)
self.dependent(drive.Touch([self.drive_folder_name])).submit()
if isinstance(self.spec.aoi, ee.FeatureCollection):
feature_collection = self.spec.aoi
aois = [
ee.Feature(
feature_collection.filterMetadata(
'system:index', 'equals',
feature_index).first()).geometry()
for feature_index in get_info(
feature_collection.aggregate_array('system:index'))
]
else:
aois = [self.spec.aoi]
self.download_tasks = [
DownloadFeature(drive_folder=self.drive_folder_name,
download_dir=self.download_dir,
description=self.description,
feature_description=str(i + 1).zfill(
len(str(len(aois)))),
spec=self.spec._replace(aoi=aoi))
for i, aoi in enumerate(aois)
]
return Task.submit_all(self.download_tasks) \
.then(self.resolve, self.reject)
def run(self):
logger.info('Monitoring Earth Engine export. task_id: {0}, destination: {1}'
.format(self.task_id, self.destination))
ee.InitializeThread(self.credentials)
while self.running():
time.sleep(10)
self.check()
def before():
credentials = AccessTokenCredentials.create()
if not credentials:
credentials = gee.service_account_credentials
logger.debug('Using credentials: ' + str(credentials))
thread_local.context = Context(credentials=credentials, download_dir=sys.argv[3])
ee.InitializeThread(credentials)
def before():
credentials = gee.service_account_credentials
if path.exists(access_token_file):
credentials = FileCredentials(access_token_file)
logger.debug('Using credentials: ' + str(credentials))
thread_local.context = Context(credentials=credentials,
download_dir=sys.argv[3])
ee.InitializeThread(credentials)
def init_ee():
credentials = service_account_credentials
if 'sepal-user' in request.headers:
user = json.loads(request.headers['sepal-user'])
googleTokens = user.get('googleTokens', None)
if googleTokens:
credentials = Credentials(googleTokens['accessToken'])
ee.InitializeThread(credentials)
def test_timeseries():
ee.InitializeThread(credentials)
#
# class CancelingTask(ThreadTask):
# def __init__(self, to_stop):
# super(CancelingTask, self).__init__()
# self.to_stop = to_stop
#
# def run(self):
# print('Running stopping task')
# time.sleep(2)
# self.to_stop.cancel()
#
# print('Stopping task')
# self.resolve()
#
#
# class StatusMonitor(ThreadTask):
# def __init__(self, to_monitor):
# super(StatusMonitor, self).__init__()
# self.to_monitor = to_monitor
#
# def run(self):
# previous_status = None
# i = 0
# while self.to_monitor.running() and self.running():
# i += 1
# # if i > 60:
# # raise Exception('Test with a failure')
# status = self.to_monitor.status_message()
# if previous_status != status:
# previous_status = status
# print(status)
# time.sleep(0.1)
# self.resolve()
#
# def close(self):
# self.to_monitor.cancel()
class PrintingTask(ProcessTask):
def __init__(self):
super(PrintingTask, self).__init__()
def run(self):
raise AttributeError('A raised error')
# for i in range(0, 5):
# print('Running')
# time.sleep(1)
# self.cancel()
# self.resolve()
def close(self):
print('closing printing task')
printing_task = PrintingTask()
Task.submit_all([printing_task]).catch(re_raise).get()
def run(self):
logger.info('Exporting Earth Engine table to asset. asset_id: {0}'
.format(self.asset_id))
ee.InitializeThread(self.credentials)
task_id = self._table_to_asset()
self.dependent(
MonitorEarthEngineExportTask(self.credentials, task_id, self.asset_id)) \
.submit() \
.then(self.resolve, self.reject)
def before():
credentials = sepal.service_account_credentials
if path.exists(access_token_file):
credentials = FileCredentials(access_token_file)
else:
logging.info('Access token file not found: ' + access_token_file)
logging.info('Using credentials: ' + str(credentials))
thread_local.credentials = credentials
ee.InitializeThread(credentials)
def run(self):
logger.info(
'Exporting Earth Engine table to cloud storage. description: {0}, folder: {1}'
.format(self.description, self.folder))
ee.InitializeThread(self.credentials)
task_id = self._table_to_cloud_storage()
self.dependent(
MonitorEarthEngineExportTask(self.credentials, task_id, self.description)) \
.submit() \
.then(self.resolve, self.reject)
def run(self):
ee.InitializeThread(self._spec.credentials)
time_series = TimeSeries(self._spec)
stack = time_series.stack
dates = time_series.dates
if not get_info(dates.size()):
return self.resolve()
self._table_export = self.dependent(
TableToDrive(
credentials=self._spec.credentials,
table=dates,
description='_'.join([self._description, self._feature_description, str(self._year), 'dates']),
folder=self._drive_folder
))
self._table_download = self.dependent(
Download(
credentials=self._spec.credentials,
drive_path=self._drive_folder,
destination_path=self._year_dir,
matching='{0}/{1}*.csv'.format(self._drive_folder, self._table_export.description),
move=True
))
self._image_export = self.dependent(
ImageToDrive(
credentials=self._spec.credentials,
image=stack,
region=self._spec.aoi,
description='_'.join([self._description, self._feature_description, str(self._year), 'stack']),
folder=self._drive_folder,
scale=30,
maxPixels=1e12,
shardSize=64,
fileDimensions=512
))
self._image_download = self.dependent(
Download(
credentials=self._spec.credentials,
drive_path=self._drive_folder,
destination_path=self._year_dir,
matching='{0}/{1}*.tif'.format(self._drive_folder, self._image_export.description),
move=True
))
self._process_year = self.dependent(
ProcessYear(self._year_dir))
Task.submit_all([
self._table_export.submit()
.then(self._table_download.submit, self.reject).catch(self.reject),
self._image_export.submit()
.then(self._image_download.submit, self.reject).catch(self.reject)
]) \
.then(self._process_year.submit, self.reject) \
.then(self.resolve, self.reject)
def cancel():
if task.state in [
Task.State.COMPLETED, Task.State.CANCEL_REQUESTED,
Task.State.CANCELLED, Task.State.FAILED
]:
return
ee.InitializeThread(credentials)
try:
task.cancel()
except Exception:
logging.exception('{}: cancelling failed'.format(task))
return empty()
def run(self):
ee.InitializeThread(self.credentials)
image_spec = image_spec_factory.create(self.image_spec)
self._export = self.dependent(
ImageToAsset(credentials=self.credentials,
image=image_spec._ee_image(),
region=image_spec.aoi.geometry(),
description=self.description,
scale=image_spec.scale))
return self._export.submit() \
.then(self.resolve, self.reject)
def run(self):
ee.InitializeThread(self.credentials)
create_asset_image_collection(
to_asset_id(self.asset_path + '/primitives'))
create_asset_image_collection(
to_asset_id(self.asset_path + '/smoothed-primitives'))
create_asset_image_collection(
to_asset_id(self.asset_path + '/assembly'))
self.drive_folder = drive.create_folder(self.credentials,
self.drive_folder_name)
return self.pipe(self._sample, self._create_primitives, self._smooth,
self._assemble, self.resolve)
def close(self, retry=0):
try:
ee.InitializeThread(self.credentials)
task = find_task(self.task_id)
if task and task.state in (Task.State.READY, Task.State.RUNNING):
logger.debug('Canceling Earth Engine Task {0}: {1}'.format(self, task.status()))
task.cancel()
except ee.EEException as e:
logger.warn('Failed to cancel task {0}: {1}'.format(self, e))
if retry < 3:
throttle_seconds = max(2 ^ retry * random.uniform(0.1, 0.2), 30)
time.sleep(throttle_seconds)
self.close(retry + 1)
def create(spec, context):
credentials = context.credentials
ee.InitializeThread(credentials)
image_spec = image_spec_factory.create(context.sepal_api, spec['image'])
return export_image_to_asset(
credentials=credentials,
description=spec['description'],
image=image_spec._ee_image(),
pyramiding_policy=image_spec.pyramiding_policy,
dimensions=None,
scale=image_spec.scale,
crs='EPSG:4326',
max_pixels=1e12
)
def index():
if not accessToken:
return redirect(
'https://accounts.google.com/o/oauth2/v2/auth?'
'scope=' + ee.oauth.SCOPE +
' https://www.googleapis.com/auth/drive&'
'redirect_uri=http%3A%2F%2F127.0.0.1:5001%2Foauth%2Fcallback&'
'response_type=code&'
'client_id=' + oauth['web']['client_id'])
credentials = OAuth2Credentials(accessToken, None, None, None, None, None,
None)
ee.InitializeThread(credentials)
return 'Hello, GEE!'
def before():
credentials = AccessTokenCredentials.create(earthengine_credentials_file)
if not credentials:
credentials = gee.service_account_credentials
logger.info('Using credentials: ' + str(credentials))
thread_local.context = Context(
credentials=credentials,
download_dir=download_dir,
username=username,
sepal_api=SepalApi(host=sepal_host,
username=sepal_username,
password=sepal_password),
)
ee.InitializeThread(credentials)
def close(self):
try:
ee.InitializeThread(self.credentials)
status = Task(self.task_id).status()
if status['state'] in (Task.State.READY, Task.State.RUNNING):
logger.debug('Canceling Earth Engine Task {0}: {1}'.format(
self, status))
Task(
status['id'], {
'type': status['task_type'],
'description': status['description'],
'state': status['state'],
}).cancel()
except ee.EEException as e:
logger.warn('Failed to cancel task {0}: {1}'.format(self, e))
def create(spec, context):
credentials = context.credentials
ee.InitializeThread(credentials)
image_spec = image_spec_factory.create(context.sepal_api, spec['image'])
return image_to_sepal(
credentials,
description=spec['description'],
download_dir=context.download_dir,
image=image_spec._ee_image(),
dimensions=None,
scale=image_spec.scale,
crs='EPSG:4326',
max_pixels=1e12,
shard_size=256,
file_dimensions=4096
)
def _poll_status(self):
ee.InitializeThread(self.credentials)
logging.debug('Starting polling status of Google Earth Engine export task: ' + self.task_id)
try:
while self.running:
status = self._task_status()
self.listener.update_status(status)
if status['state'] != 'ACTIVE':
self.stop()
return
time.sleep(10)
except Exception:
logger.exception('Export to Google Drive failed. Task id: ' + self.task_id)
self.listener.update_status({
'state': 'FAILED',
'description': 'Export to Google Drive failed'})
self.stop()
def initialize(ee_account='', ee_key_path='', ee_user_token=''):
try:
if ee_user_token:
credentials = OAuth2Credentials(ee_user_token, None, None, None,
None, None, None)
ee.InitializeThread(credentials)
elif ee_account and ee_key_path:
credentials = ServiceAccountCredentials.from_p12_keyfile(
service_account_email=ee_account,
filename=ee_key_path,
private_key_password='******',
scopes=ee.oauth.SCOPE +
' https://www.googleapis.com/auth/drive')
ee.Initialize(credentials)
else:
ee.Initialize()
except (EEException, TypeError):
pass
def _export_year(geometry, year_start, year_end, export_description,
year_dir):
stack = _create_stack(geometry, year_start, year_end)
ee.InitializeThread(credentials)
if not stack.bandNames().size().getInfo():
logging.info('No data between {} and {}'.format(
year_start, year_end))
return of({
'exported': 1,
'downloaded': 1,
'downloaded_bytes': 0,
'processed': 1
})
initial_progress = of({
'exported': 0,
'stack_bytes': 0,
'dates_bytes': 0,
'downloaded': 0,
'processed': 0
})
def aggregate_downloaded_bytes(p):
return {
'exported': p['exported'],
'downloaded': p['downloaded'],
'downloaded_bytes': p['stack_bytes'] + p['dates_bytes'],
'processed': p['processed']
}
return concat(
initial_progress,
merge(
_export_and_download_stack(stack, export_description,
year_dir),
_export_and_download_dates(stack, export_description,
year_dir)), _process_year(year_dir),
of({'processed': 1})).pipe(scan(lambda acc, p: {
**acc,
**p
}, {}), map(aggregate_downloaded_bytes))
def run(self):
ee.InitializeThread(self.credentials)
self._drive_folder = drive.create_folder(self.credentials, self.drive_path)
self.dependent(drive.Touch([self.drive_path], self.credentials)).submit()
image_spec = image_spec_factory.create(self.sepal_api, self.image_spec)
self._export = self.dependent(
ImageToDrive(
credentials=self.credentials,
image=image_spec._ee_image(),
region=image_spec.aoi.geometry(),
description=self.description,
folder=self.drive_path,
scale=image_spec.scale,
maxPixels=1e12,
shardSize=256,
fileDimensions=4096
))
destination_path = self.download_dir + '/' + self.description
self._download = self.dependent(
Download(
credentials=self.credentials,
drive_path=self.drive_path,
destination_path=destination_path,
move=True
))
tifs = destination_path + '/*.tif'
self._build_vrt = self.dependent(
BuildVrt(destination_path + '/' + self.description + '.vrt', tifs))
self._set_band_names = self.dependent(
SetBandNames(image_spec.bands, [tifs, destination_path + '/*.vrt']))
self._build_overviews = self.dependent(
BuildOverviews(destination_path + '/*.vrt'))
return self._export.submit() \
.then(self._download.submit, self.reject) \
.then(self._build_vrt.submit, self.reject) \
.then(self._set_band_names.submit, self.reject) \
.then(self._build_overviews.submit, self.reject) \
.then(self.resolve, self.reject)
def schedule():
credentials = get_credentials()
return rx.interval(period, TimeoutScheduler()).pipe(
do_action(lambda _: ee.InitializeThread(credentials)),
)
def create(spec, context):
def get_region():
aoi_spec = spec['aoi']
if aoi_spec['type'] == 'FUSION_TABLE':
if aoi_spec.get('keyColumn'):
return Aoi.create(aoi_spec).feature_collection
else:
return ee.FeatureCollection('ft:' + aoi_spec['id'])
else:
return Aoi.create(aoi_spec).geometry()
def image_collection_factory(geometry, start_date, end_date):
if 'SENTINEL_1' in spec['dataSets']:
return create_radar_collection(geometry, start_date, end_date)
else:
return create_optical_collection(geometry, start_date, end_date)
def create_radar_collection(geometry, start_date, end_date):
collection = radar_collection.create(
region=geometry,
orbits=spec.get('orbits', ('ASCENDING', )),
start_date=start_date,
end_date=end_date,
geometric_correction=spec.get('geometricCorrection', 'ELLIPSOID'),
speckle_filter=spec.get('speckleFilter', 'NONE'),
outlier_removal=spec.get('outlierRemoval', 'NONE'),
harmonics_dependents=())
if spec['indicator'] in ['VV', 'VH']:
return collection.select(spec['indicator'])
else:
return collection.map(lambda image: replace(
image,
image.select('VV').divide(image.select('VH'))))
def create_optical_collection(geometry, start_date, end_date):
OpticalSpec = namedtuple(
'OpticalSpec',
'credentials, surface_reflectance, shadow_tolerance, '
'mask_clouds, mask_snow, brdf_correct, target_day, masked_on_analysis, bands, calibrate, cloud_buffer'
)
def _process_collection(collection, optical_spec):
collection = mask_clouds(optical_spec, collection)
collection = mask_shadows(optical_spec, collection)
return collection
optical_spec = OpticalSpec(
credentials=credentials,
surface_reflectance=spec['surfaceReflectance'],
shadow_tolerance=1,
mask_clouds=True,
mask_snow=spec['maskSnow'],
brdf_correct=spec['brdfCorrect'],
target_day=0,
masked_on_analysis=False,
bands=[],
calibrate='SENTINEL_2' in spec['dataSets'],
cloud_buffer=0)
def calculate_indicator(image):
return replace(
image,
optical_indexes.to_index(
image, spec['indicator']).multiply(10000).int16())
def get_data_sets():
date_filter = ee.Filter.date(to_ee_date(start_date),
to_ee_date(end_date))
landsat_data_set_names = [
name for name in spec['dataSets'] if name.startswith('LANDSAT')
]
data_sets = landsat_data_sets(landsat_data_set_names, geometry,
optical_spec, date_filter)
if 'SENTINEL_2' in spec['dataSets']:
data_sets.append(
sentinel_2_data_set(geometry, optical_spec, date_filter))
return data_sets
data_sets = get_data_sets()
collection = ee.ImageCollection([])
for data_set in data_sets:
data_set_collection = analyze(optical_spec, data_set,
data_set.to_collection())
collection = ee.ImageCollection(
collection.merge(data_set_collection))
return _process_collection(collection, optical_spec) \
.map(calculate_indicator)
credentials = context.credentials
ee.InitializeThread(credentials)
return time_series_to_sepal(
credentials,
description=spec['description'],
download_dir=context.download_dir,
image_collection_factory=image_collection_factory,
start_date=spec['fromDate'],
end_date=spec['toDate'],
region=get_region(),
scale=spec.get('scale', 30),
crs='EPSG:4326',
max_pixels=1e12,
file_dimensions=512,
nodata_value=0,
retries=2)
def image_to_sepal(
credentials,
description: str,
download_dir: str,
image: ee.Image,
band_names: list = None,
dimensions=None,
region: ee.Geometry = None,
scale: int = None,
crs: str = None,
crs_transform: str = None,
max_pixels: Union[int, float] = None,
shard_size: int = None,
file_dimensions=None,
skip_empty_tiles=None,
file_format: str = None,
format_options: str = None,
retries: int = 0
):
drive_folder_path = '_'.join(['Sepal', description, str(uuid.uuid4())])
destination_path = download_dir + '/' + description
def _create_drive_folder():
return concat(
progress(
default_message='Creating Google Drive download folder...',
message_key='tasks.retrieve.image_to_sepal.creating_drive_folder'
),
create_folder_with_path(credentials, drive_folder_path).pipe(
flat_map(lambda _: empty())
)
)
def _export_to_drive():
return export_image_to_drive(
credentials,
image,
description=description,
folder=drive_folder_path,
dimensions=dimensions,
region=region,
scale=scale,
crs=crs,
crs_transform=crs_transform,
max_pixels=max_pixels,
shard_size=shard_size,
file_dimensions=file_dimensions,
skip_empty_tiles=skip_empty_tiles,
file_format=file_format,
format_options=format_options,
retries=retries,
)
def _download_from_drive():
return download_path(
credentials,
path=drive_folder_path,
destination=destination_path,
delete_after_download=True
)
def _delete_drive_folder():
return concat(
progress(
default_message='Deleting Google Drive download folder...',
message_key='tasks.retrieve.image_to_sepal.deleting_drive_folder'
),
delete_file_with_path(
credentials,
path=drive_folder_path
).pipe(
flat_map(lambda _: empty())
)
)
def _build_vrt():
return concat(
progress(
default_message='Building VRT...',
message_key='tasks.retrieve.image_to_sepal.building_vrt'
),
build_vrt(
destination=destination_path + '/' + description + '.vrt',
files=destination_path + '/*.tif'
).pipe(
flat_map(lambda _: empty())
)
)
def _set_band_names():
band_names_stream = of(band_names) if band_names else get_band_names(credentials, image)
return concat(
progress(
default_message='Setting band names...',
message_key='tasks.retrieve.image_to_sepal.setting_band_names'
),
band_names_stream.pipe(
flat_map(
lambda names: set_band_names(
band_names=names,
files=[destination_path + '/*.tif', destination_path + '/*.vrt']
)
),
flat_map(lambda _: empty())
)
)
ee.InitializeThread(credentials)
return concat(
_create_drive_folder(),
_export_to_drive(),
_download_from_drive(),
_delete_drive_folder(),
_build_vrt(),
_set_band_names()
).pipe(
merge_finalize(_delete_drive_folder)
)
def run(self):
ee.InitializeThread(self.credentials)
return self._create_primitive(self.samples).pipe(self.resolve)
