def update_recipe(task: Task, recipe):
errors = []
deep_update(recipe, RECIPE_CLUSTER_TPL)
files = []
try:
path = task.get_stack_path(full=True)
except AssertionError as e:
errors.append(str(e))
return errors
if os.path.isdir(path):
# raise AssertionError(f"Task stack directory '{path}' not found")
files = os.listdir(path)
__resolve_files(FILE_PATTERN, files, recipe)
recipe['PREDICTOR_DIR'] = task.config['predictor']
recipe['COS']['bucket'] = task.config['cos_bucket']
try:
with open(task.get_predictor_config_file_name(), 'r') as _f:
j = json.load(_f)
deep_update(recipe, j) # DO NOT INLINE j
# pprint(self.recipe)
except JSONDecodeError as e:
errors.append(f'Predictor config JSON is invalid, reason: {e}')
except (RuntimeError, AssertionError) as e:
errors.append(str(e))
# pprint(recipe)
return errors
def task_stack_snap(task: Task, dry_run, gpt_cache,cmd_dir,log):
# TODO http://remote-sensing.eu/preprocessing-of-sentinel-1-sar-data-via-snappy-python-module/
""" Run master-slave Stacking
"""
snap_path = task.get_stack_path(full=True)
log(f"Using ESA SNAP processing pipeline in {snap_path}")
os.makedirs(snap_path, exist_ok=True)
cmd = os.path.join(cmd_dir, 'local-snap.sh')
_eodata = task.config['eodata']
_docker_mount = 'mnt'
opts = [
cmd,
'--gpt-cache', gpt_cache,
'--eodata', _eodata,
'--snap_results', snap_path,
'--swath', task.config['swath'],
'--firstBurstIndex', task.config['firstBurstIndex'],
'--lastBurstIndex', task.config['lastBurstIndex'],
'--master', _local_eodata_relative_path(_eodata, task.config['master_path']),
'--slave', _local_eodata_relative_path(_eodata, task.config['slave_path']),
]
if dry_run:
log("Command:")
opts = opts + ['--dry-run']
opts = opts + [' '] # add space to the end for booleans
# print(opts)
# print(" ".join(opts))
subprocess.run(opts)
def ai_preview_stack(repo: Repo, task: Task, roi_id, recipe_path, slice_range,
show_list,
# rgb,
band, columns, clip, hist, save, export, ylog):
""" Preview assembled tensor band
** use --clip <minl> <max> to apply np.log10(np.clip(.., 10**min, 10**max)) to stack values
\b
* Windows WSL: follow https://www.scivision.dev/pyqtmatplotlib-in-windows-subsystem-for-linux/
"""
try:
_recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id)
recipe = Recipe(_recipe)
_dir = recipe.get('DATADIR')
except (RuntimeError, AssertionError, click.UsageError) as e:
output.comment(f"Could not resolve recipe {e}, fall-back to task")
try:
_dir = task.get_stack_path('snap_path')
except AssertionError as e:
raise click.UsageError(f'Could not get stack path: {e}')
except Exception as e:
log.exception("Could not resolve Stack results")
raise click.UsageError('Could not resolve Stack results')
output.comment(f"Stack dir: {_dir}\n\n")
full_shape, df = get_stack_df(_dir)
if show_list:
output.table(df[['filename', 'resolution', 'path']], showindex=True,
headers=['band', 'name', 'resolution', 'path'])
else:
try:
# if rgb:
# if len(rgb) != 3:
# raise AssertionError('rgb', '--rgb should contain exactly 3 digits without spaces')
# band = (int(rgb[0]), int(rgb[1]), int(rgb[2]))
if band[0] == -1:
band = list(range(0, len(df)))
else:
band = list(band)
_ds = df.iloc[band] # type: gpd.GeoDataFrame
output.table(_ds, showindex=True)
_plt = preview_stack(_ds, _dir,
full_shape=full_shape,
slice_region=slice_range,
band=band,
clip=clip,
columns=columns,
hist=hist,
ylog=ylog
)
_show_plt(_plt, save=save)
except AssertionError as e:
log.exception(e)
raise click.UsageError(str(e))
def ai_preview_stack_math(repo: Repo, task: Task, roi_id, recipe_path, slice_range, show_list,
band1, band2, band3,
vis_mode, data_path, save, export, hist, ylog):
""" Band math for stack
{common}
"""
if not data_path:
try:
_recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id)
recipe = Recipe(_recipe)
data_path = recipe.get('DATADIR')
output.comment(f'Using recipe file "{recipe_path}"')
except (RuntimeError, AssertionError, click.UsageError) as e:
output.comment(f'Using stack from task stack_results')
try:
data_path = task.get_stack_path('snap_path')
if not os.path.isdir(data_path):
raise AssertionError(f'Directory "{data_path}" is not exists ')
except AssertionError as e:
raise click.UsageError(f'Could not get stack_results: {e}')
output.comment(f"Stack dir: {data_path}\n\n")
full_shape, df = get_stack_df(data_path)
if show_list:
output.table(df, showindex=True)
else:
title, (r, g, b) = create_stack_rgb(band1, band2, band3,
df=df,
vis_mode=vis_mode,
slice_range=slice_range,
)
if export:
georef = df.iloc[band1].path
_save_envi_rgb(r, g, b, export=export,
georef=georef, data_path=data_path, slice_range=slice_range,
title=title
)
else:
_plt = _vis_rgb(r, g, b, title, hist, ylog)
_show_plt(_plt, save)