def to_progress(state): if state == 'PENDING': return progress(default_message= 'Submitting export task to Google Earth Engine...', message_key='tasks.ee.export.pending') elif state == 'READY': return progress( default_message= 'Waiting for Google Earth Engine to start export...', message_key='tasks.ee.export.ready') elif state == 'RUNNING': return progress( default_message='Google Earth Engine is exporting...', message_key='tasks.ee.export.running') else: return empty()
def _create_drive_folder(): return concat( progress( default_message='Creating Google Drive download folder...', message_key= 'tasks.retrieve.time_series_to_sepal.creating_drive_folder'), create_folder_with_path(credentials, drive_folder_path).pipe( flat_map(lambda _: empty())))
def _build_vrt(): return concat( progress(default_message='Building VRT...', message_key='tasks.retrieve.image_to_sepal.building_vrt'), build_vrt(destination=destination_path + '/' + description + '.vrt', files=destination_path + '/*.tif').pipe( flat_map(lambda _: empty())))
def _delete_drive_folder(): return concat( progress( default_message='Deleting Google Drive download folder...', message_key= 'tasks.retrieve.image_to_sepal.deleting_drive_folder'), delete_file_with_path(credentials, path=drive_folder_path).pipe( flat_map(lambda _: empty())))
def download_file(f, dest): total_bytes = int(f['size']) def next_chunk(downloader): status, done = downloader.next_chunk() logging.debug('downloaded chunk from {} to {}: {}'.format( file, destination, status)) return 1.0 if done else status.progress() def download_from_drive(destination_file): def create_downloader(): request = get_service(credentials).files().get_media( fileId=f['id']) return MediaIoBaseDownload(fd=destination_file, request=request, chunksize=CHUNK_SIZE) downloader = create_downloader() return forever().pipe( map(lambda _: next_chunk(downloader)), take_while(lambda p: p < 1, inclusive=True), flat_map(lambda p: progress( default_message= 'Downloaded {downloaded_files} of {total_files} files ({downloaded} of {total})', message_key='tasks.drive.download_folder', downloaded_bytes=int(total_bytes * p), downloaded=format_bytes(int(total_bytes * p)), total_bytes=total_bytes, total=format_bytes(total_bytes), file=f))) def action(): return using_file(file=dest, mode='wb', to_observable=download_from_drive) os.makedirs(os.path.dirname(dest), exist_ok=True) initial_progress = progress( default_message= 'Downloaded {downloaded_files} of {total_files} files ({downloaded} of {total})', message_key='tasks.drive.download_folder', downloaded_bytes=0, downloaded='0 bytes', total_bytes=total_bytes, total=format_bytes(total_bytes), file=f) touch_stream = interval(TOUCH_PERIOD).pipe( flat_map(lambda _: touch(credentials, f))) download_stream = enqueue(credentials, queue=_drive_downloads, action=action, retries=retries, description='Download {} to {}'.format( f, dest)).pipe(aside(touch_stream)) return concat(initial_progress, download_stream, delete_downloaded(f))
def _export(credentials, create_observable, description, retries): return concat( progress( default_message='Submitting export task to Google Earth Engine...', message_key='tasks.ee.export.pending'), enqueue(credentials, queue=_ee_exports, action=create_observable, description=description, retries=retries))
def _export_geometries(): def aggregate_progress(progresses, count): p = _sum_dicts(progresses.values(), excluded_keys=['geometry']) exported = round(100 * p['exported'] / count) downloaded = round(100 * p['downloaded'] / count) downloaded_bytes = format_bytes(p['downloaded_bytes']) processed = round(100 * p['processed'] / count) return progress( default_message='Exported {}%, Downloaded {}% ({}), Processed {}%'.format( exported, downloaded, downloaded_bytes, processed ), message_key='tasks.retrieve.time_series_to_sepal.progress', exported=exported, downloaded=downloaded, downloaded_bytes=downloaded_bytes, processed=processed ) features_collection = _to_features_collection(region) def export_geometry(geometry, i, geometry_count): geometry_description = str(i + 1).zfill(len(str(geometry_count))) return defer( lambda _: _export_geometry( geometry, geometry_description=geometry_description ) ) return concat( progress( default_message='Tiling AOI...', message_key='tasks.retrieve.time_series_to_sepal.tiling' ), _extract_feature_indexes(features_collection).pipe( flat_map( lambda feature_indexes: _to_geometries(features_collection, feature_indexes).pipe( flat_map( lambda geometries: concat( *[ export_geometry(geometry, i, len(feature_indexes)) for i, geometry in enumerate(geometries) ] ) ), scan(lambda acc, p: {**acc, p['geometry']: p}, {}), flat_map(lambda progresses: aggregate_progress( progresses, count=len(feature_indexes) * len(year_ranges) )) ) ) ) )
def _set_band_names(): band_names_stream = of(band_names) if band_names else get_band_names( credentials, image) return concat( progress( default_message='Setting band names...', message_key='tasks.retrieve.image_to_sepal.setting_band_names' ), band_names_stream.pipe( flat_map(lambda names: set_band_names( band_names=names, files= [destination_path + '/*.tif', destination_path + '/*.vrt']) ), flat_map(lambda _: empty())))
def aggregate_progress(progresses, count): p = _sum_dicts(progresses.values(), excluded_keys=['geometry']) exported = round(100 * p['exported'] / count) downloaded = round(100 * p['downloaded'] / count) downloaded_bytes = format_bytes(p['downloaded_bytes']) processed = round(100 * p['processed'] / count) return progress( default_message= 'Exported {}%, Downloaded {}% ({}), Processed {}%'.format( exported, downloaded, downloaded_bytes, processed), message_key='tasks.retrieve.time_series_to_sepal.progress', exported=exported, downloaded=downloaded, downloaded_bytes=downloaded_bytes, processed=processed)
def aggregate_progress(progresses: list): total_files = len(progresses) total_bytes = sum([int(p.file['size']) for p in progresses]) downloaded_files = len( [p for p in progresses if p.downloaded_bytes == p.total_bytes]) downloaded_bytes = sum([p.downloaded_bytes for p in progresses]) return progress( default_message= 'Downloaded {downloaded_files} of {total_files} files ({downloaded} of {total})', message_key='tasks.drive.download_folder', downloaded_files=downloaded_files, downloaded_bytes=downloaded_bytes, downloaded=format_bytes(downloaded_bytes), total_files=total_files, total_bytes=total_bytes, total=format_bytes(total_bytes))
def download_from_drive(destination_file): def create_downloader(): request = get_service(credentials).files().get_media( fileId=f['id']) return MediaIoBaseDownload(fd=destination_file, request=request, chunksize=CHUNK_SIZE) downloader = create_downloader() return forever().pipe( map(lambda _: next_chunk(downloader)), take_while(lambda p: p < 1, inclusive=True), flat_map(lambda p: progress( default_message= 'Downloaded {downloaded_files} of {total_files} files ({downloaded} of {total})', message_key='tasks.drive.download_folder', downloaded_bytes=int(total_bytes * p), downloaded=format_bytes(int(total_bytes * p)), total_bytes=total_bytes, total=format_bytes(total_bytes), file=f)))