def ai_assemble(repo: Repo, task: Task, roi_id, recipe_path: str, zone: str,
fast):
""" assemble tensor from co-registered stack by given recipe
[zone|full]- assemble tensor from full image or from the part defined by "zone" key in recipe JSON
if no --recipe provided, recipe will be taken based on active task
"""
_recipe = recipe_path if recipe_path else resolve_recipe(
repo, task, roi_id)
recipe = Recipe(_recipe)
try:
cos = COS(recipe)
except SystemExit:
log.warning("Could not use COS")
output.warning("Could not use COS")
cos = None
envi = Envi(recipe, cos)
log.info('Assembling tensor')
assembler = Assemble(zone, recipe, envi)
if repo.verbose == 'DEBUG':
assembler.run()
else:
with pfac(log, total=100, desc='Assembling') as (_, callback):
try:
assembler.run(callback)
except AssertionError as e:
raise click.UsageError(f'{e}')
except Exception as e:
log.exception(e)
raise click.UsageError(f'{e}')
def _resolve_tensor_filenames(repo, task, zone, roi_id, data_path, recipe_path, tnorm) -> Filenames:
if not data_path:
try:
_recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id)
recipe = Recipe(_recipe)
output.comment(f'Using recipe file "{_recipe}"')
except (RuntimeError, AssertionError, click.UsageError) as e:
output.comment(f'Using tensor from ai_results')
try:
data_path = task.get_ai_results_path(full=True)
if not os.path.isdir(data_path):
raise AssertionError(f'Directory "{data_path}" is not exists ')
recipe = {'OUTDIR': data_path}
except AssertionError as e:
raise click.UsageError(f'Could not get ai_results: {e}')
else:
recipe = {'OUTDIR': data_path}
if tnorm and 'PREDICTOR_DIR' not in recipe:
try:
_filenames = Filenames(zone, recipe)
with open(_filenames.process_info, 'r') as f:
_prcinfo = yaml.load(f, Loader=yaml.FullLoader)
recipe['PREDICTOR_DIR'] = _prcinfo['process']['PREDICTOR_DIR']
except Exception as e:
raise OCLIException(f"Could not resolve tnorm file: {e}")
return Filenames(zone, recipe)
def ai_upload(repo: Repo, task: Task, roi_id, recipe_path, cos_key, dry_run):
"""Upload COG TIFF to cloud storage"""
_recipe = recipe_path if recipe_path else resolve_recipe(
repo, task, roi_id)
recipe = Recipe(_recipe)
filenames = Filenames('zone', recipe)
cog_file = filenames.out_cog_tiff
doc_json = Path(filenames.out_cog_tiff + '.geojson')
if not doc_json.is_file():
raise OCLIException(f'Could not find "{doc_json.absolute()}"')
_json = open(doc_json, 'r').read()
try:
doc = json.loads(_json)
except JSONDecodeError:
raise OCLIException(f'Could not parse json "{doc_json.absolute()}"')
if not cos_key and "ResultKey" in doc['properties']:
cos_key = doc['properties'].get('ResultKey')
if not cos_key:
raise click.UsageError("No COS key (upload file name)")
if not cos_key.endswith('.tiff'):
cos_key += '.tiff'
log.info(
f"About to upload {cog_file} as {cos_key} to bucket {recipe['COS'].get('bucket')} "
)
try:
cos = COS(recipe)
except SystemExit:
raise click.UsageError(
f'Invalid recipe: COS credentials in "{_recipe}" are required for upload'
)
try:
if dry_run:
output.comment(
f'Uploading "{cog_file}" as "{cos_key}" into bucket "{cos.bucket}"'
)
output.comment(
f'Uploading "{cog_file}.geojson" as "{cos_key}.geojson" into bucket "{cos.bucket}"'
)
else:
filesize = os.stat(cog_file).st_size
with tqdm(total=filesize, unit='B', unit_scale=True,
desc=cos_key) as t:
cos.upload_to_cos(cog_file, cos_key, hook(t))
if os.path.isfile(cog_file + '.geojson'):
filesize = os.stat(cog_file + '.geojson').st_size
with tqdm(total=filesize,
unit='B',
unit_scale=True,
desc=cos_key + '.geojson') as t:
cos.upload_to_cos(cog_file + '.geojson',
cos_key + '.geojson', hook(t))
except SystemExit as e:
raise click.UsageError(e)
def ai_preview_stack(repo: Repo, task: Task, roi_id, recipe_path, slice_range,
show_list,
# rgb,
band, columns, clip, hist, save, export, ylog):
""" Preview assembled tensor band
** use --clip <minl> <max> to apply np.log10(np.clip(.., 10**min, 10**max)) to stack values
\b
* Windows WSL: follow https://www.scivision.dev/pyqtmatplotlib-in-windows-subsystem-for-linux/
"""
try:
_recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id)
recipe = Recipe(_recipe)
_dir = recipe.get('DATADIR')
except (RuntimeError, AssertionError, click.UsageError) as e:
output.comment(f"Could not resolve recipe {e}, fall-back to task")
try:
_dir = task.get_stack_path('snap_path')
except AssertionError as e:
raise click.UsageError(f'Could not get stack path: {e}')
except Exception as e:
log.exception("Could not resolve Stack results")
raise click.UsageError('Could not resolve Stack results')
output.comment(f"Stack dir: {_dir}\n\n")
full_shape, df = get_stack_df(_dir)
if show_list:
output.table(df[['filename', 'resolution', 'path']], showindex=True,
headers=['band', 'name', 'resolution', 'path'])
else:
try:
# if rgb:
# if len(rgb) != 3:
# raise AssertionError('rgb', '--rgb should contain exactly 3 digits without spaces')
# band = (int(rgb[0]), int(rgb[1]), int(rgb[2]))
if band[0] == -1:
band = list(range(0, len(df)))
else:
band = list(band)
_ds = df.iloc[band] # type: gpd.GeoDataFrame
output.table(_ds, showindex=True)
_plt = preview_stack(_ds, _dir,
full_shape=full_shape,
slice_region=slice_range,
band=band,
clip=clip,
columns=columns,
hist=hist,
ylog=ylog
)
_show_plt(_plt, save=save)
except AssertionError as e:
log.exception(e)
raise click.UsageError(str(e))
def ai_visualize(repo: Repo, task: Task, roi_id, recipe_path, zone):
"""visualize AI processing results"""
try:
_recipe = recipe_path if recipe_path else resolve_recipe(
repo, task, roi_id)
recipe = Recipe(_recipe)
try:
cos = COS(recipe)
except SystemExit:
log.warning("Could not use COS")
output.warning("Could not use COS")
cos = None
envi = Envi(recipe, cos)
Visualize(zone, recipe, envi).run()
except AssertionError as e:
raise click.BadArgumentUsage(str(e))
def ai_preview_cluster(repo: Repo, task: Task, roi_id, recipe_path, slice_range, show_list, band, columns,
# threshold,
zone,
hist, ylog,
save, export,
rgb=False
):
""" Preview assembled tensor band
\b
* Windows WSL: follow https://www.scivision.dev/pyqtmatplotlib-in-windows-subsystem-for-linux/
"""
try:
_recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id)
recipe = Recipe(_recipe)
filenames = Filenames(zone, recipe)
pred8c_img = filenames.pred8c_img
pred8c_hdr = filenames.pred8c_hdr
if not os.path.isfile(pred8c_img):
raise AssertionError(f"IMG file '{pred8c_img}' not fond")
if not os.path.isfile(pred8c_hdr):
raise AssertionError(f"HDR file '{pred8c_hdr}' not fond")
pred8c_hdr = s_envi.open(filenames.pred8c_hdr)
except (AssertionError) as e:
raise click.UsageError(f"Could not visualize: {e}")
if show_list:
output.comment(f'Cluster HDR: {filenames.pred8c_hdr}')
x, y = pred8c_hdr.shape[:2]
bn = pred8c_hdr.metadata['band names']
bn = [[b, f'{x}x{y}'] for b in bn]
output.table(bn, showindex=True, headers=['band', 'name', 'resolution'])
return
# if rgb:
# if len(rgb) != 3:
# raise click.BadOptionUsage('rgb', '--rgb should contain exactly 3 digits without spaces')
# band = (int(rgb[0]), int(rgb[1]), int(rgb[2]))
if band[0] == -1:
band = list(range(0, pred8c_hdr.shape[2]))
preview_cluster(filenames.pred8c_hdr, filenames.pred8c_img,
band=band,
slice_region=slice_range,
columns=columns,
rgb=rgb
)
def ai_preview_stack_math(repo: Repo, task: Task, roi_id, recipe_path, slice_range, show_list,
band1, band2, band3,
vis_mode, data_path, save, export, hist, ylog):
""" Band math for stack
{common}
"""
if not data_path:
try:
_recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id)
recipe = Recipe(_recipe)
data_path = recipe.get('DATADIR')
output.comment(f'Using recipe file "{recipe_path}"')
except (RuntimeError, AssertionError, click.UsageError) as e:
output.comment(f'Using stack from task stack_results')
try:
data_path = task.get_stack_path('snap_path')
if not os.path.isdir(data_path):
raise AssertionError(f'Directory "{data_path}" is not exists ')
except AssertionError as e:
raise click.UsageError(f'Could not get stack_results: {e}')
output.comment(f"Stack dir: {data_path}\n\n")
full_shape, df = get_stack_df(data_path)
if show_list:
output.table(df, showindex=True)
else:
title, (r, g, b) = create_stack_rgb(band1, band2, band3,
df=df,
vis_mode=vis_mode,
slice_range=slice_range,
)
if export:
georef = df.iloc[band1].path
_save_envi_rgb(r, g, b, export=export,
georef=georef, data_path=data_path, slice_range=slice_range,
title=title
)
else:
_plt = _vis_rgb(r, g, b, title, hist, ylog)
_show_plt(_plt, save)
def ai_predict(repo: Repo, task: Task, roi_id, recipe_path, zone, pred_type):
"""Run cluster analysis on assembled tensor
\b
* zone- process full tensor
* full- process part of the tensor as defined by "zone" key in a recipe JSON file
* fit - run cluster analysis processing only to generate predictor files
* fitpredict - run cluster analysis learning and fit
* predict - run cluster analysis based on provided in JSON recipe predictor
if no --recipe provided, recipe will be taken based on active task
"""
_recipe = recipe_path if recipe_path else resolve_recipe(
repo, task, roi_id)
recipe = Recipe(_recipe)
with pfac(
log,
total=100,
# show_eta=True,
# item_show_func=lambda x: str(x),
desc='Processing') as (_, callback):
Process(zone, pred_type, recipe).run(callback=callback)
pass
def ai_makecog(repo: Repo, task: Task, roi_id, recipe_path, json_only, quiet,
no_color, less, zone, kind, source, cos_key, friendly_name,
print_res, warp_resampleAlg, overview_resampleAlg):
""" Make COG TIFF from visualized results
\b
* to make GeoTIFF from custom ENVI file use --source option with filename of ENVI file (without extension)
* to override recipe kind, use --kind option,
example: making image from 'ai preview --export path/to/envi' file use
makecog zone --kind Image --source path/to/envi
* to avoid overriding recipe main results use --cos-key and --friendly-name option
if --friendly-name starts with '+' value will be used as suffix for friendly_name in GeoJSON
if --cos-key starts with '+' value will be used as suffix for COS.ResultKey in GeoJSON
"""
driver = 'MAKECOG'
if source:
try:
# Only valid ENVI or GeoTiff files are alloed
str = gdal.Info(source, format='json')
driver = str['driverShortName']
if driver not in ['ENVI', 'GTiff']:
raise OCLIException(
f"Unsupported source file type: {str['driverShortName']} ({str['driverLongName']})"
)
except Exception as e:
raise OCLIException(f"Option --source: {e}")
_recipe = recipe_path if recipe_path else resolve_recipe(
repo, task, roi_id)
recipe = Recipe(_recipe)
kind = kind if kind != 'auto' else recipe.get('type')
if kind in ['Rvi', 'Image']:
recipe['type'] = 'Image'
# log.error(recipe['COS']['ResultKey'])
if cos_key:
if cos_key.startswith('+'):
recipe['COS']['ResultKey'] += cos_key[1:]
else:
recipe['COS']['ResultKey'] = cos_key
if friendly_name:
if friendly_name.startswith('+'):
recipe['friendly_name'] += friendly_name[1:]
else:
recipe['friendly_name'] = friendly_name
# log.error(recipe['COS']['ResultKey'])
# log.error(recipe['friendly_name'])
filenames = Filenames(zone, recipe)
if source:
input_file = source
else:
input_file = filenames.pred8c_img
out_file = filenames.out_tiff
cog_file = filenames.out_cog_tiff
check_file = cog_file if json_only else input_file
if not Path(check_file).is_file():
raise OCLIException(f'file not found: {check_file}')
os.makedirs(Path(cog_file).parent, exist_ok=True)
w = GDALWrap3(recipe, input_file, out_file, cog_file)
try:
if not json_only:
if driver == 'GTiff':
try:
shutil.copy(input_file, cog_file)
output.success(f"file {input_file} copied to {cog_file}")
except Exception as e:
raise OCLIException(
f'Could not copy "{input_file}" to "{cog_file}" : {e}'
)
else:
if not quiet:
with pfac(log, total=100,
desc='Assembling') as (_, callback):
def cb(pct, msg, user_data):
_ud = user_data[0]
# self.log.debug(f"translaing: {round(pct * 100, 2)}% -{msg}- {user_data} {pct}")
# reset counter, this t\callback called from multiple prcesses
if pct < 0.01:
user_data[0] = 0
if user_data[0] == 0 or pct - _ud > 0.10:
log.debug(
f"Local translating : {round(pct * 100, 2)}%"
)
user_data[0] = pct
callback(100, pct, 'translating')
w.make_cog(cb, warp_resampleAlg, overview_resampleAlg)
else:
w.make_cog(None, warp_resampleAlg, overview_resampleAlg)
_json = w.make_geo_json()
_json = _json if no_color else colorful_json(_json)
if print_res:
click.echo("\n\n\n")
if less:
click.echo_via_pager(_json)
else:
click.echo(_json)
except (RuntimeError, OCLIException) as e:
raise click.UsageError(
output.error_style(f"COG-tiff generation failed,reason: {e}"))
pass