def download_order(order_id, dest, quiet, pretty):
'''Download an order by given order ID'''
cl = clientv1()
dl = downloader.create(cl, order=True)
output = downloader_output(dl, disable_ansi=quiet)
output.start()
items = cl.get_individual_order(order_id).items_iter(limit=None)
handle_interrupt(dl.shutdown, dl.download, items, [], dest)
def download(asset_type, dest, limit, sort, search_id, dry_run, activate_only,
quiet, **kw):
'''Activate and download'''
cl = clientv1()
page_size = min(limit or 250, 250)
asset_type = list(chain.from_iterable(asset_type))
# even though we're using functionality from click.Path, this was needed
# to detect inability to write on Windows in a read-only vagrant mount...
# @todo check/report upstream
if not activate_only and not check_writable(dest):
raise click.ClickException(
'download destination "%s" is not writable' % dest)
if search_id:
if dry_run:
raise click.ClickException(
'dry-run not supported with saved search')
if any(kw[s] for s in kw):
raise click.ClickException(
'search options not supported with saved search')
search, search_arg = cl.saved_search, search_id
else:
# any requested asset-types should be used as permission filters
kw['asset_type'] = [AssetTypePerm.to_permissions(asset_type)]
req = search_req_from_opts(**kw)
if dry_run:
req['interval'] = 'year'
stats = cl.stats(req).get()
item_cnt = sum([b['count'] for b in stats['buckets']])
asset_cnt = item_cnt * len(asset_type)
click.echo(
'would download approximately %d assets from %s items' %
(asset_cnt, item_cnt)
)
return
else:
search, search_arg = cl.quick_search, req
dl = downloader.create(cl)
output = downloader_output(dl, disable_ansi=quiet)
# delay initial item search until downloader output initialized
output.start()
try:
items = search(search_arg, page_size=page_size, sort=sort)
except Exception as ex:
output.cancel()
click_exception(ex)
func = dl.activate if activate_only else dl.download
args = [items.items_iter(limit), asset_type]
if not activate_only:
args.append(dest)
# invoke the function within an interrupt handler that will shut everything
# down properly
handle_interrupt(dl.shutdown, func, *args)
def download_quads(name, rbox, quiet, dest, limit):
'''Download quads from a mosaic'''
cl = clientv1()
dl = downloader.create(cl, mosaic=True)
output = downloader_output(dl, disable_ansi=quiet)
output.start()
try:
mosaic, = cl.get_mosaic_by_name(name).items_iter(1)
items = cl.get_quads(mosaic, rbox).items_iter(limit)
except Exception as ex:
output.cancel()
click_exception(ex)
# invoke the function within an interrupt handler that will shut everything
# down properly
handle_interrupt(dl.shutdown, dl.download, items, [], dest)
def test_pipeline():
logging.basicConfig(
stream=sys.stderr, level=logging.INFO,
format='%(asctime)s %(message)s', datefmt='%M:%S'
)
cl = HelperClient()
items = items_iter(100)
asset_types = ['a', 'b']
dl = downloader.create(
cl, no_sleep=True,
astage__size=10, pstage__size=10, pstage__min_poll_interval=0,
dstage__size=2)
completed = []
dl.on_complete = lambda *a: completed.append(a)
stats = handle_interrupt(dl.shutdown, dl.download, items,
asset_types, 'dest')
assert stats == {
'downloading': 0, 'complete': 200, 'paging': False,
'downloaded': '0.20MB', 'activating': 0, 'pending': 0
}
assert 200 == len(completed)
