def enqueue(credentials,
queue: WorkQueue,
mapper: Mapper,
description: str = None,
retries: int = 0):
return rx.pipe(
flat_map(lambda value: queue.enqueue(observable=defer(lambda _: mapper(
value)),
group=str(credentials),
description=description,
retries=retries)))
def enqueue(credentials,
queue: WorkQueue,
mapper: Mapper,
description: str = None,
retries: int = 0):
def mapper_to_observable(value):
ee.InitializeThread(credentials)
return mapper(value)
return rx.pipe(
flat_map(lambda value: queue.enqueue(observable=defer(
lambda _: mapper_to_observable(value)),
group=str(credentials),
description=description,
retries=0)),
retry(credentials, retries, description))
def enqueue(queue: WorkQueue,
mapper: Mapper,
description: str = None,
retries: int = None):
credentials = get_credentials()
def mapper_to_observable(value):
def initializing_mapper():
ee.InitializeThread(credentials)
return mapper(value)
return from_callable(initializing_mapper, _ee_executions.scheduler)
return rx.pipe(
flat_map(
lambda value: queue.enqueue(observable=mapper_to_observable(value),
group=str(credentials),
description=description,
retries=0)),
retry(retries, description))
from glob import glob
from typing import Union
from osgeo import gdal
from osgeo.gdalconst import GA_Update
from rx import Observable, defer, from_callable
from sepal.rx.workqueue import WorkQueue
gdal_queue = WorkQueue(2)
def set_band_names(band_names: list, files: Union[str, list]) -> Observable:
def action():
if isinstance(files, str):
files_ = [files]
else:
files_ = files
globbed_files = []
for f in files_:
globbed_files += glob(f)
for f in globbed_files:
ds = gdal.Open(f, GA_Update)
for i in range(0, len(band_names)):
band = ds.GetRasterBand(i + 1)
band.SetMetadata({'BAND_NAME': band_names[i]})
ds.FlushCache()
return defer(lambda _: gdal_queue.enqueue(
from_callable(action),
retries=3,
lambda task: delete_asset(credentials, task.config['assetId']).
pipe(flat_map(lambda _: execute_task(credentials, task)))))
return _export(credentials,
create_observable=create_observable,
description=description,
retries=retries)
def _export_to_drive(credentials, create_task, description, retries):
return _export(
credentials,
create_observable=lambda: execute_task(credentials, create_task()),
description=description,
retries=retries)
# Work (i.e. exports) is grouped by credentials, limiting concurrent exports per credentials
_ee_exports = WorkQueue(concurrency_per_group=2,
description='earth-engine-exports')
def _export(credentials, create_observable, description, retries):
return concat(
of('PENDING'),
enqueue(credentials,
queue=_ee_exports,
action=create_observable,
description=description,
retries=retries))
from sepal.drive import get_service, is_folder
from sepal.drive.rx.list import list_folder_recursively
from sepal.format import format_bytes
from sepal.rx import aside, forever, using_file
from sepal.rx.workqueue import WorkQueue
from sepal.task.rx.observables import progress
from .delete import delete_file
from .observables import enqueue
from .touch import touch
CHUNK_SIZE = 100 * 1024 * 1024
TOUCH_PERIOD = 15 * 60 # Every 15 minutes - to prevent it from being garbage collected from Service Account
# Work (i.e. exports) is grouped by credentials, limiting concurrent exports per credentials
_drive_downloads = WorkQueue(concurrency_per_group=2,
description='drive-exports')
def download(credentials,
file: dict,
destination: str,
matching: str = None,
delete_after_download: bool = False,
retries: int = 5) -> Observable:
logging.debug('downloading {} to {}'.format(file, destination))
destination = os.path.abspath(destination)
def get_file_destination(f):
relative_path = f['path'][len(file['path']):]
next_destination = '{}{}{}'.format(
destination, '' if destination[-1] == '/' else '/', relative_path)
description: str = None,
retries: int = 0):
def mapper_to_observable(value):
ee.InitializeThread(credentials)
return mapper(value)
return rx.pipe(
flat_map(lambda value: queue.enqueue(observable=defer(
lambda _: mapper_to_observable(value)),
group=str(credentials),
description=description,
retries=0)),
retry(credentials, retries, description))
# A single group is used to execute EE operations, independent of user and credentials used.
_ee_executions = WorkQueue(concurrency_per_group=5,
description='earth-engine-actions')
def execute(credentials,
mapper: Mapper = None,
retries: int = 3,
description: str = None):
return rx.pipe(
enqueue(credentials,
queue=_ee_executions,
mapper=lambda value: from_callable(lambda: mapper(value)),
description=description,
retries=retries))
def enqueue(credentials,
queue: WorkQueue,
mapper: Mapper,
description: str = None,
retries: int = 0):
return rx.pipe(
flat_map(lambda value: queue.enqueue(observable=defer(lambda _: mapper(
value)),
group=str(credentials),
description=description,
retries=retries)))
_drive_actions = WorkQueue(concurrency_per_group=2,
delay_seconds=.1,
description='drive-actions')
def execute(credentials,
mapper: Mapper = None,
retries: int = 3,
description: str = None):
return rx.pipe(
enqueue(credentials,
queue=_drive_actions,
mapper=lambda value: from_callable(lambda: mapper(value)),
description=description,
retries=retries))