def download(ctx, tracking_name, update_, clean_, dest_dir, overwrite):
""" Download pre-ARD for a tracked order
Downloads data into a directory named after the order name. Pass
``--dest`` to change the root location of this directory.
\b
TODO
----
* Silence / don't use progressbar if we're quiet
"""
from cedar.utils import load_ee
config = options.fetch_config(ctx)
tracker = config.get_tracker()
# Rename to remove .json
tracking_name = tracking_name.rstrip('.json')
# Destination defaults to tracking_name
if dest_dir is None:
# remove any extension listed
dest_dir = Path('.').resolve()
else:
dest_dir = Path(dest_dir)
dest = dest_dir.joinpath(tracking_name)
click.echo(f'Retrieving pre-ARD from tracking info: {tracking_name}')
if update_:
load_ee(True) # authenticate against EE API
tracking_info = tracker.update(tracking_name)
else:
tracking_info = tracker.read(tracking_name)
n_tasks = len(tracking_info['orders'])
click.echo(f'Downloading data for {n_tasks} tasks')
with click.progressbar(label='Downloading',
item_show_func=_item_show_func,
length=n_tasks) as bar:
cb_bar = _make_callback(bar)
dl_info = tracker.download(tracking_info,
dest,
overwrite=overwrite,
callback=cb_bar)
if clean_:
if tracking_info.complete:
clean_info = _do_clean(tracker, tracking_name, tracking_info,
False)
else:
click.echo('Not cleaning data because the order is still in '
'progress or the tracking metadata has not been '
f'updated. Run ``cedar status update {tracking_name}`` '
'if you believe the order has completed')
click.echo('Complete!')
def cancel(ctx, tracking_name):
from ..utils import get_ee_tasks, load_ee
ee_api = load_ee(True)
logger = ctx.obj['logger']
config = options.fetch_config(ctx)
tracker = config.get_tracker()
# Read tracking info
info = tracker.read(tracking_name)
# Cancel tasks
for task in info.tasks:
if task is not None:
task.cancel()
click.echo(f'Cancelled task ID "{task.id}"')
else:
click.echo('Skipping task that may be unsubmitted '
'or already expired')
# Delete tracking
tracker.clean(info, tracking_name=tracking_name)
if logger.level <= logging.WARNING:
click.echo('Complete')
def update(ctx, tracking_name, all_, dest):
""" Update job submission tracking info
"""
if all_ and tracking_name:
raise click.BadParameter('Cannot specify tracking names AND `--all`',
param_hint='--all')
from cedar.utils import load_ee
ee = load_ee(True)
logger = ctx.obj['logger']
config = options.fetch_config(ctx)
tracker = config.get_tracker()
if all_:
tracking_name = tracker.list()
else:
tracking_name = [tracking_name]
dest = Path(dest) if dest else None
for name in tracking_name:
if logger.level <= logging.WARNING:
click.echo(f'Updating "{name}"')
info = tracker.update(name)
if dest:
dest.mkdir(exist_ok=True, parents=True)
dest_ = dest.joinpath(name)
with open(str(dest_), 'w') as dst:
json.dump(info, dst, indent=2)
if logger.level <= logging.WARNING:
click.echo('Complete')
def auth_gee(ctx):
""" Test logging into the Google Earth Engine
If it doesn't work, please make sure you've authenticated by running
>>> earthengine authenticate
"""
from cedar.utils import load_ee
try:
ee = load_ee(True)
except Exception as e:
click.echo('Could not load and initialize the Earth Engine API')
click.echo(e)
raise click.Abort()
else:
click.echo(options.STYLE_INFO('Authenticated'))
def tasks(ctx, list_tasks):
""" Get info about Google Earth Engine tasks
"""
from cedar.utils import get_ee_tasks, load_ee
ee = load_ee(True)
tasks = get_ee_tasks()
# Summarize
totals = defaultdict(lambda: 0)
for task in tasks.values():
totals[task.state] += 1
click.echo('Task summary:')
for state, count in totals.items():
click.echo(f' {state}: {count}')
# Print out verbose info
if list_tasks:
click.echo('Tasks:')
for id_, task in tasks.items():
click.echo(f'{id_} - {task.state} - {task.task_type}=>'
f'{task.config["description"]}')
def completed(ctx, tracking_name, update_):
""" Print percent of order completed & exit 1 if not complete
"""
from cedar.utils import load_ee
ee = load_ee(True)
logger = ctx.obj['logger']
config = options.fetch_config(ctx)
tracker = config.get_tracker()
if update_:
info = tracker.update(tracking_name)
else:
info = tracker.read(tracking_name)
percent = info.progress
if logger.level <= logging.WARNING:
click.echo(f'Percent complete: {percent * 100. :03.2f}')
if percent == 1.0:
ctx.exit(0)
else:
ctx.exit(1)
def submit(ctx, image_collection, index, row, col, period_start, period_end,
period_freq, date_format):
""" Submit "pre-ARD" processing orders and create tracking metadata
"""
from cedar.exceptions import EmptyOrderError
from cedar.sensors import CREATE_ARD_COLLECTION
from cedar.utils import load_ee
# Need index OR (row AND col) -- blame index param if it goes wrong
_, index_param = options.fetch_param(ctx, 'index')
if index:
if row or col:
raise click.BadParameter(
'Cannot use `--index` with `--row` or `--col`',
ctx=ctx,
param=index_param)
elif row and col:
if index:
raise click.BadParameter(
'Cannot use `--row` and `--col` with `--index`',
ctx=ctx,
param=index_param)
index = list(itertools.product(row, col))
else:
raise click.BadParameter(
'Must specify `--index` OR both `--row` and `--col`',
ctx=ctx,
param=index_param)
# Eventually we'll support not having a period_start/period_end, but not yet
if not period_start or not period_end:
name = f'period_{"start" if not period_start else "end"}'
param = options.fetch_param(ctx, name)[1]
raise click.BadParameter(f'Must pass {name}ing period',
ctx=ctx,
param=param)
# Check that we know about the image collection
for collection in image_collection:
if collection not in CREATE_ARD_COLLECTION:
raise KeyError(f'Unknown image collection "{collection}"')
# Get parse(d) config and build tracker
config = options.fetch_config(ctx)
tracker = config.get_tracker()
# Login to EE
ee = load_ee(True)
msg = [
f'Tiles: {", ".join([f"h{c:04d}v{r:04d}" for r, c in index])}',
f'Collections: {", ".join(image_collection)}',
f'Time period: {period_start.isoformat()} - {period_end.isoformat()}',
f'At frequency: {period_freq}'
]
click.echo('Submitting preARD tasks for:')
click.echo("\n".join([f" {s}" for s in msg]))
# Submit!
try:
tracking_info_name, tracking_info_id = tracker.submit(
image_collection,
index,
period_start,
period_end,
period_freq=period_freq)
click.echo('Wrote job tracking to store object named '
f'"{tracking_info_name}" ({tracking_info_id})')
except EmptyOrderError as ece:
click.echo(repr(ece))
click.echo('Did not find data to order. Check your search parameters')
raise click.Abort()
except Exception as e:
click.echo('Unknown error occurred. See exception printed below')
raise e