def to_progress(state):
if state == 'PENDING':
return progress(default_message=
'Submitting export task to Google Earth Engine...',
message_key='tasks.ee.export.pending')
elif state == 'READY':
return progress(
default_message=
'Waiting for Google Earth Engine to start export...',
message_key='tasks.ee.export.ready')
elif state == 'RUNNING':
return progress(
default_message='Google Earth Engine is exporting...',
message_key='tasks.ee.export.running')
else:
return empty()
def _create_drive_folder():
return concat(
progress(
default_message='Creating Google Drive download folder...',
message_key=
'tasks.retrieve.time_series_to_sepal.creating_drive_folder'),
create_folder_with_path(credentials, drive_folder_path).pipe(
flat_map(lambda _: empty())))
def _build_vrt():
return concat(
progress(default_message='Building VRT...',
message_key='tasks.retrieve.image_to_sepal.building_vrt'),
build_vrt(destination=destination_path + '/' + description +
'.vrt',
files=destination_path + '/*.tif').pipe(
flat_map(lambda _: empty())))
def _delete_drive_folder():
return concat(
progress(
default_message='Deleting Google Drive download folder...',
message_key=
'tasks.retrieve.image_to_sepal.deleting_drive_folder'),
delete_file_with_path(credentials, path=drive_folder_path).pipe(
flat_map(lambda _: empty())))
def download_file(f, dest):
total_bytes = int(f['size'])
def next_chunk(downloader):
status, done = downloader.next_chunk()
logging.debug('downloaded chunk from {} to {}: {}'.format(
file, destination, status))
return 1.0 if done else status.progress()
def download_from_drive(destination_file):
def create_downloader():
request = get_service(credentials).files().get_media(
fileId=f['id'])
return MediaIoBaseDownload(fd=destination_file,
request=request,
chunksize=CHUNK_SIZE)
downloader = create_downloader()
return forever().pipe(
map(lambda _: next_chunk(downloader)),
take_while(lambda p: p < 1, inclusive=True),
flat_map(lambda p: progress(
default_message=
'Downloaded {downloaded_files} of {total_files} files ({downloaded} of {total})',
message_key='tasks.drive.download_folder',
downloaded_bytes=int(total_bytes * p),
downloaded=format_bytes(int(total_bytes * p)),
total_bytes=total_bytes,
total=format_bytes(total_bytes),
file=f)))
def action():
return using_file(file=dest,
mode='wb',
to_observable=download_from_drive)
os.makedirs(os.path.dirname(dest), exist_ok=True)
initial_progress = progress(
default_message=
'Downloaded {downloaded_files} of {total_files} files ({downloaded} of {total})',
message_key='tasks.drive.download_folder',
downloaded_bytes=0,
downloaded='0 bytes',
total_bytes=total_bytes,
total=format_bytes(total_bytes),
file=f)
touch_stream = interval(TOUCH_PERIOD).pipe(
flat_map(lambda _: touch(credentials, f)))
download_stream = enqueue(credentials,
queue=_drive_downloads,
action=action,
retries=retries,
description='Download {} to {}'.format(
f, dest)).pipe(aside(touch_stream))
return concat(initial_progress, download_stream, delete_downloaded(f))
def _export(credentials, create_observable, description, retries):
return concat(
progress(
default_message='Submitting export task to Google Earth Engine...',
message_key='tasks.ee.export.pending'),
enqueue(credentials,
queue=_ee_exports,
action=create_observable,
description=description,
retries=retries))
def _export_geometries():
def aggregate_progress(progresses, count):
p = _sum_dicts(progresses.values(), excluded_keys=['geometry'])
exported = round(100 * p['exported'] / count)
downloaded = round(100 * p['downloaded'] / count)
downloaded_bytes = format_bytes(p['downloaded_bytes'])
processed = round(100 * p['processed'] / count)
return progress(
default_message='Exported {}%, Downloaded {}% ({}), Processed {}%'.format(
exported, downloaded, downloaded_bytes, processed
),
message_key='tasks.retrieve.time_series_to_sepal.progress',
exported=exported,
downloaded=downloaded,
downloaded_bytes=downloaded_bytes,
processed=processed
)
features_collection = _to_features_collection(region)
def export_geometry(geometry, i, geometry_count):
geometry_description = str(i + 1).zfill(len(str(geometry_count)))
return defer(
lambda _: _export_geometry(
geometry,
geometry_description=geometry_description
)
)
return concat(
progress(
default_message='Tiling AOI...',
message_key='tasks.retrieve.time_series_to_sepal.tiling'
),
_extract_feature_indexes(features_collection).pipe(
flat_map(
lambda feature_indexes: _to_geometries(features_collection, feature_indexes).pipe(
flat_map(
lambda geometries: concat(
*[
export_geometry(geometry, i, len(feature_indexes))
for i, geometry in enumerate(geometries)
]
)
),
scan(lambda acc, p: {**acc, p['geometry']: p}, {}),
flat_map(lambda progresses: aggregate_progress(
progresses,
count=len(feature_indexes) * len(year_ranges)
))
)
)
)
)
def _set_band_names():
band_names_stream = of(band_names) if band_names else get_band_names(
credentials, image)
return concat(
progress(
default_message='Setting band names...',
message_key='tasks.retrieve.image_to_sepal.setting_band_names'
),
band_names_stream.pipe(
flat_map(lambda names: set_band_names(
band_names=names,
files=
[destination_path + '/*.tif', destination_path + '/*.vrt'])
), flat_map(lambda _: empty())))
def aggregate_progress(progresses, count):
p = _sum_dicts(progresses.values(), excluded_keys=['geometry'])
exported = round(100 * p['exported'] / count)
downloaded = round(100 * p['downloaded'] / count)
downloaded_bytes = format_bytes(p['downloaded_bytes'])
processed = round(100 * p['processed'] / count)
return progress(
default_message=
'Exported {}%, Downloaded {}% ({}), Processed {}%'.format(
exported, downloaded, downloaded_bytes, processed),
message_key='tasks.retrieve.time_series_to_sepal.progress',
exported=exported,
downloaded=downloaded,
downloaded_bytes=downloaded_bytes,
processed=processed)
def aggregate_progress(progresses: list):
total_files = len(progresses)
total_bytes = sum([int(p.file['size']) for p in progresses])
downloaded_files = len(
[p for p in progresses if p.downloaded_bytes == p.total_bytes])
downloaded_bytes = sum([p.downloaded_bytes for p in progresses])
return progress(
default_message=
'Downloaded {downloaded_files} of {total_files} files ({downloaded} of {total})',
message_key='tasks.drive.download_folder',
downloaded_files=downloaded_files,
downloaded_bytes=downloaded_bytes,
downloaded=format_bytes(downloaded_bytes),
total_files=total_files,
total_bytes=total_bytes,
total=format_bytes(total_bytes))
def download_from_drive(destination_file):
def create_downloader():
request = get_service(credentials).files().get_media(
fileId=f['id'])
return MediaIoBaseDownload(fd=destination_file,
request=request,
chunksize=CHUNK_SIZE)
downloader = create_downloader()
return forever().pipe(
map(lambda _: next_chunk(downloader)),
take_while(lambda p: p < 1, inclusive=True),
flat_map(lambda p: progress(
default_message=
'Downloaded {downloaded_files} of {total_files} files ({downloaded} of {total})',
message_key='tasks.drive.download_folder',
downloaded_bytes=int(total_bytes * p),
downloaded=format_bytes(int(total_bytes * p)),
total_bytes=total_bytes,
total=format_bytes(total_bytes),
file=f)))
