def download(ctx, tracking_name, update_, clean_, dest_dir, overwrite): """ Download pre-ARD for a tracked order Downloads data into a directory named after the order name. Pass ``--dest`` to change the root location of this directory. \b TODO ---- * Silence / don't use progressbar if we're quiet """ from cedar.utils import load_ee config = options.fetch_config(ctx) tracker = config.get_tracker() # Rename to remove .json tracking_name = tracking_name.rstrip('.json') # Destination defaults to tracking_name if dest_dir is None: # remove any extension listed dest_dir = Path('.').resolve() else: dest_dir = Path(dest_dir) dest = dest_dir.joinpath(tracking_name) click.echo(f'Retrieving pre-ARD from tracking info: {tracking_name}') if update_: load_ee(True) # authenticate against EE API tracking_info = tracker.update(tracking_name) else: tracking_info = tracker.read(tracking_name) n_tasks = len(tracking_info['orders']) click.echo(f'Downloading data for {n_tasks} tasks') with click.progressbar(label='Downloading', item_show_func=_item_show_func, length=n_tasks) as bar: cb_bar = _make_callback(bar) dl_info = tracker.download(tracking_info, dest, overwrite=overwrite, callback=cb_bar) if clean_: if tracking_info.complete: clean_info = _do_clean(tracker, tracking_name, tracking_info, False) else: click.echo('Not cleaning data because the order is still in ' 'progress or the tracking metadata has not been ' f'updated. Run ``cedar status update {tracking_name}`` ' 'if you believe the order has completed') click.echo('Complete!')
def cancel(ctx, tracking_name): from ..utils import get_ee_tasks, load_ee ee_api = load_ee(True) logger = ctx.obj['logger'] config = options.fetch_config(ctx) tracker = config.get_tracker() # Read tracking info info = tracker.read(tracking_name) # Cancel tasks for task in info.tasks: if task is not None: task.cancel() click.echo(f'Cancelled task ID "{task.id}"') else: click.echo('Skipping task that may be unsubmitted ' 'or already expired') # Delete tracking tracker.clean(info, tracking_name=tracking_name) if logger.level <= logging.WARNING: click.echo('Complete')
def update(ctx, tracking_name, all_, dest): """ Update job submission tracking info """ if all_ and tracking_name: raise click.BadParameter('Cannot specify tracking names AND `--all`', param_hint='--all') from cedar.utils import load_ee ee = load_ee(True) logger = ctx.obj['logger'] config = options.fetch_config(ctx) tracker = config.get_tracker() if all_: tracking_name = tracker.list() else: tracking_name = [tracking_name] dest = Path(dest) if dest else None for name in tracking_name: if logger.level <= logging.WARNING: click.echo(f'Updating "{name}"') info = tracker.update(name) if dest: dest.mkdir(exist_ok=True, parents=True) dest_ = dest.joinpath(name) with open(str(dest_), 'w') as dst: json.dump(info, dst, indent=2) if logger.level <= logging.WARNING: click.echo('Complete')
def auth_gee(ctx): """ Test logging into the Google Earth Engine If it doesn't work, please make sure you've authenticated by running >>> earthengine authenticate """ from cedar.utils import load_ee try: ee = load_ee(True) except Exception as e: click.echo('Could not load and initialize the Earth Engine API') click.echo(e) raise click.Abort() else: click.echo(options.STYLE_INFO('Authenticated'))
def tasks(ctx, list_tasks): """ Get info about Google Earth Engine tasks """ from cedar.utils import get_ee_tasks, load_ee ee = load_ee(True) tasks = get_ee_tasks() # Summarize totals = defaultdict(lambda: 0) for task in tasks.values(): totals[task.state] += 1 click.echo('Task summary:') for state, count in totals.items(): click.echo(f' {state}: {count}') # Print out verbose info if list_tasks: click.echo('Tasks:') for id_, task in tasks.items(): click.echo(f'{id_} - {task.state} - {task.task_type}=>' f'{task.config["description"]}')
def completed(ctx, tracking_name, update_): """ Print percent of order completed & exit 1 if not complete """ from cedar.utils import load_ee ee = load_ee(True) logger = ctx.obj['logger'] config = options.fetch_config(ctx) tracker = config.get_tracker() if update_: info = tracker.update(tracking_name) else: info = tracker.read(tracking_name) percent = info.progress if logger.level <= logging.WARNING: click.echo(f'Percent complete: {percent * 100. :03.2f}') if percent == 1.0: ctx.exit(0) else: ctx.exit(1)
def submit(ctx, image_collection, index, row, col, period_start, period_end, period_freq, date_format): """ Submit "pre-ARD" processing orders and create tracking metadata """ from cedar.exceptions import EmptyOrderError from cedar.sensors import CREATE_ARD_COLLECTION from cedar.utils import load_ee # Need index OR (row AND col) -- blame index param if it goes wrong _, index_param = options.fetch_param(ctx, 'index') if index: if row or col: raise click.BadParameter( 'Cannot use `--index` with `--row` or `--col`', ctx=ctx, param=index_param) elif row and col: if index: raise click.BadParameter( 'Cannot use `--row` and `--col` with `--index`', ctx=ctx, param=index_param) index = list(itertools.product(row, col)) else: raise click.BadParameter( 'Must specify `--index` OR both `--row` and `--col`', ctx=ctx, param=index_param) # Eventually we'll support not having a period_start/period_end, but not yet if not period_start or not period_end: name = f'period_{"start" if not period_start else "end"}' param = options.fetch_param(ctx, name)[1] raise click.BadParameter(f'Must pass {name}ing period', ctx=ctx, param=param) # Check that we know about the image collection for collection in image_collection: if collection not in CREATE_ARD_COLLECTION: raise KeyError(f'Unknown image collection "{collection}"') # Get parse(d) config and build tracker config = options.fetch_config(ctx) tracker = config.get_tracker() # Login to EE ee = load_ee(True) msg = [ f'Tiles: {", ".join([f"h{c:04d}v{r:04d}" for r, c in index])}', f'Collections: {", ".join(image_collection)}', f'Time period: {period_start.isoformat()} - {period_end.isoformat()}', f'At frequency: {period_freq}' ] click.echo('Submitting preARD tasks for:') click.echo("\n".join([f" {s}" for s in msg])) # Submit! try: tracking_info_name, tracking_info_id = tracker.submit( image_collection, index, period_start, period_end, period_freq=period_freq) click.echo('Wrote job tracking to store object named ' f'"{tracking_info_name}" ({tracking_info_id})') except EmptyOrderError as ece: click.echo(repr(ece)) click.echo('Did not find data to order. Check your search parameters') raise click.Abort() except Exception as e: click.echo('Unknown error occurred. See exception printed below') raise e