def download_order(order_id, dest, quiet, pretty): '''Download an order by given order ID''' cl = clientv1() dl = downloader.create(cl, order=True) output = downloader_output(dl, disable_ansi=quiet) output.start() items = cl.get_individual_order(order_id).items_iter(limit=None) handle_interrupt(dl.shutdown, dl.download, items, [], dest)
def download(asset_type, dest, limit, sort, search_id, dry_run, activate_only, quiet, **kw): '''Activate and download''' cl = clientv1() page_size = min(limit or 250, 250) asset_type = list(chain.from_iterable(asset_type)) # even though we're using functionality from click.Path, this was needed # to detect inability to write on Windows in a read-only vagrant mount... # @todo check/report upstream if not activate_only and not check_writable(dest): raise click.ClickException( 'download destination "%s" is not writable' % dest) if search_id: if dry_run: raise click.ClickException( 'dry-run not supported with saved search') if any(kw[s] for s in kw): raise click.ClickException( 'search options not supported with saved search') search, search_arg = cl.saved_search, search_id else: # any requested asset-types should be used as permission filters kw['asset_type'] = [AssetTypePerm.to_permissions(asset_type)] req = search_req_from_opts(**kw) if dry_run: req['interval'] = 'year' stats = cl.stats(req).get() item_cnt = sum([b['count'] for b in stats['buckets']]) asset_cnt = item_cnt * len(asset_type) click.echo( 'would download approximately %d assets from %s items' % (asset_cnt, item_cnt) ) return else: search, search_arg = cl.quick_search, req dl = downloader.create(cl) output = downloader_output(dl, disable_ansi=quiet) # delay initial item search until downloader output initialized output.start() try: items = search(search_arg, page_size=page_size, sort=sort) except Exception as ex: output.cancel() click_exception(ex) func = dl.activate if activate_only else dl.download args = [items.items_iter(limit), asset_type] if not activate_only: args.append(dest) # invoke the function within an interrupt handler that will shut everything # down properly handle_interrupt(dl.shutdown, func, *args)
def download_quads(name, rbox, quiet, dest, limit): '''Download quads from a mosaic''' cl = clientv1() dl = downloader.create(cl, mosaic=True) output = downloader_output(dl, disable_ansi=quiet) output.start() try: mosaic, = cl.get_mosaic_by_name(name).items_iter(1) items = cl.get_quads(mosaic, rbox).items_iter(limit) except Exception as ex: output.cancel() click_exception(ex) # invoke the function within an interrupt handler that will shut everything # down properly handle_interrupt(dl.shutdown, dl.download, items, [], dest)
def test_pipeline(): logging.basicConfig( stream=sys.stderr, level=logging.INFO, format='%(asctime)s %(message)s', datefmt='%M:%S' ) cl = HelperClient() items = items_iter(100) asset_types = ['a', 'b'] dl = downloader.create( cl, no_sleep=True, astage__size=10, pstage__size=10, pstage__min_poll_interval=0, dstage__size=2) completed = [] dl.on_complete = lambda *a: completed.append(a) stats = handle_interrupt(dl.shutdown, dl.download, items, asset_types, 'dest') assert stats == { 'downloading': 0, 'complete': 200, 'paging': False, 'downloaded': '0.20MB', 'activating': 0, 'pending': 0 } assert 200 == len(completed)