def _download_from_drive():
return download_path(
credentials,
path=drive_folder_path,
destination=destination_path,
delete_after_download=True
).pipe(
flat_map(lambda status: progress(
default_message='Downloaded {downloaded_files} of {total_files} files ({downloaded} of {total})',
message_key='tasks.retrieve.image_to_sepal.download_progress',
downloaded_files=status['downloaded_files'],
downloaded=format_bytes(status['downloaded_bytes']),
total_files=status['total_files'],
total=format_bytes(status['total_bytes'])
))
)
def aggregate_progress(progresses: list):
total_files = len(progresses)
total_bytes = sum([int(p.file['size']) for p in progresses])
downloaded_files = len(
[p for p in progresses if p.downloaded_bytes == p.total_bytes])
downloaded_bytes = sum([p.downloaded_bytes for p in progresses])
return progress(
default_message=
'Downloaded {downloaded_files} of {total_files} files ({downloaded} of {total})',
message_key='tasks.drive.download_folder',
downloaded_files=downloaded_files,
downloaded_bytes=downloaded_bytes,
downloaded=format_bytes(downloaded_bytes),
total_files=total_files,
total_bytes=total_bytes,
total=format_bytes(total_bytes))
def download_file(f, dest):
total_bytes = int(f['size'])
def next_chunk(downloader):
status, done = downloader.next_chunk()
logging.debug('downloaded chunk from {} to {}: {}'.format(
file, destination, status))
return 1.0 if done else status.progress()
def download_from_drive(destination_file):
def create_downloader():
request = get_service(credentials).files().get_media(
fileId=f['id'])
return MediaIoBaseDownload(fd=destination_file,
request=request,
chunksize=CHUNK_SIZE)
downloader = create_downloader()
return forever().pipe(
map(lambda _: next_chunk(downloader)),
take_while(lambda p: p < 1, inclusive=True),
flat_map(lambda p: progress(
default_message=
'Downloaded {downloaded_files} of {total_files} files ({downloaded} of {total})',
message_key='tasks.drive.download_folder',
downloaded_bytes=int(total_bytes * p),
downloaded=format_bytes(int(total_bytes * p)),
total_bytes=total_bytes,
total=format_bytes(total_bytes),
file=f)))
def action():
return using_file(file=dest,
mode='wb',
to_observable=download_from_drive)
os.makedirs(os.path.dirname(dest), exist_ok=True)
initial_progress = progress(
default_message=
'Downloaded {downloaded_files} of {total_files} files ({downloaded} of {total})',
message_key='tasks.drive.download_folder',
downloaded_bytes=0,
downloaded='0 bytes',
total_bytes=total_bytes,
total=format_bytes(total_bytes),
file=f)
touch_stream = interval(TOUCH_PERIOD).pipe(
flat_map(lambda _: touch(credentials, f)))
download_stream = enqueue(credentials,
queue=_drive_downloads,
action=action,
retries=retries,
description='Download {} to {}'.format(
f, dest)).pipe(aside(touch_stream))
return concat(initial_progress, download_stream, delete_downloaded(f))
def download_from_drive(destination_file):
def create_downloader():
request = get_service(credentials).files().get_media(
fileId=f['id'])
return MediaIoBaseDownload(fd=destination_file,
request=request,
chunksize=CHUNK_SIZE)
downloader = create_downloader()
return forever().pipe(
map(lambda _: next_chunk(downloader)),
take_while(lambda p: p < 1, inclusive=True),
flat_map(lambda p: progress(
default_message=
'Downloaded {downloaded_files} of {total_files} files ({downloaded} of {total})',
message_key='tasks.drive.download_folder',
downloaded_bytes=int(total_bytes * p),
downloaded=format_bytes(int(total_bytes * p)),
total_bytes=total_bytes,
total=format_bytes(total_bytes),
file=f)))
def aggregate_progress(progresses, count):
p = _sum_dicts(progresses.values(), excluded_keys=['geometry'])
exported = round(100 * p['exported'] / count)
downloaded = round(100 * p['downloaded'] / count)
downloaded_bytes = format_bytes(p['downloaded_bytes'])
processed = round(100 * p['processed'] / count)
return progress(
default_message=
'Exported {}%, Downloaded {}% ({}), Processed {}%'.format(
exported, downloaded, downloaded_bytes, processed),
message_key='tasks.retrieve.time_series_to_sepal.progress',
exported=exported,
downloaded=downloaded,
downloaded_bytes=downloaded_bytes,
processed=processed)