def ai_preview_stack(repo: Repo, task: Task, roi_id, recipe_path, slice_range, show_list, # rgb, band, columns, clip, hist, save, export, ylog): """ Preview assembled tensor band ** use --clip <minl> <max> to apply np.log10(np.clip(.., 10**min, 10**max)) to stack values \b * Windows WSL: follow https://www.scivision.dev/pyqtmatplotlib-in-windows-subsystem-for-linux/ """ try: _recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id) recipe = Recipe(_recipe) _dir = recipe.get('DATADIR') except (RuntimeError, AssertionError, click.UsageError) as e: output.comment(f"Could not resolve recipe {e}, fall-back to task") try: _dir = task.get_stack_path('snap_path') except AssertionError as e: raise click.UsageError(f'Could not get stack path: {e}') except Exception as e: log.exception("Could not resolve Stack results") raise click.UsageError('Could not resolve Stack results') output.comment(f"Stack dir: {_dir}\n\n") full_shape, df = get_stack_df(_dir) if show_list: output.table(df[['filename', 'resolution', 'path']], showindex=True, headers=['band', 'name', 'resolution', 'path']) else: try: # if rgb: # if len(rgb) != 3: # raise AssertionError('rgb', '--rgb should contain exactly 3 digits without spaces') # band = (int(rgb[0]), int(rgb[1]), int(rgb[2])) if band[0] == -1: band = list(range(0, len(df))) else: band = list(band) _ds = df.iloc[band] # type: gpd.GeoDataFrame output.table(_ds, showindex=True) _plt = preview_stack(_ds, _dir, full_shape=full_shape, slice_region=slice_range, band=band, clip=clip, columns=columns, hist=hist, ylog=ylog ) _show_plt(_plt, save=save) except AssertionError as e: log.exception(e) raise click.UsageError(str(e))
def ai_assemble(repo: Repo, task: Task, roi_id, recipe_path: str, zone: str, fast): """ assemble tensor from co-registered stack by given recipe [zone|full]- assemble tensor from full image or from the part defined by "zone" key in recipe JSON if no --recipe provided, recipe will be taken based on active task """ _recipe = recipe_path if recipe_path else resolve_recipe( repo, task, roi_id) recipe = Recipe(_recipe) try: cos = COS(recipe) except SystemExit: log.warning("Could not use COS") output.warning("Could not use COS") cos = None envi = Envi(recipe, cos) log.info('Assembling tensor') assembler = Assemble(zone, recipe, envi) if repo.verbose == 'DEBUG': assembler.run() else: with pfac(log, total=100, desc='Assembling') as (_, callback): try: assembler.run(callback) except AssertionError as e: raise click.UsageError(f'{e}') except Exception as e: log.exception(e) raise click.UsageError(f'{e}')
def _resolve_tensor_filenames(repo, task, zone, roi_id, data_path, recipe_path, tnorm) -> Filenames: if not data_path: try: _recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id) recipe = Recipe(_recipe) output.comment(f'Using recipe file "{_recipe}"') except (RuntimeError, AssertionError, click.UsageError) as e: output.comment(f'Using tensor from ai_results') try: data_path = task.get_ai_results_path(full=True) if not os.path.isdir(data_path): raise AssertionError(f'Directory "{data_path}" is not exists ') recipe = {'OUTDIR': data_path} except AssertionError as e: raise click.UsageError(f'Could not get ai_results: {e}') else: recipe = {'OUTDIR': data_path} if tnorm and 'PREDICTOR_DIR' not in recipe: try: _filenames = Filenames(zone, recipe) with open(_filenames.process_info, 'r') as f: _prcinfo = yaml.load(f, Loader=yaml.FullLoader) recipe['PREDICTOR_DIR'] = _prcinfo['process']['PREDICTOR_DIR'] except Exception as e: raise OCLIException(f"Could not resolve tnorm file: {e}") return Filenames(zone, recipe)
def ai_upload(repo: Repo, task: Task, roi_id, recipe_path, cos_key, dry_run): """Upload COG TIFF to cloud storage""" _recipe = recipe_path if recipe_path else resolve_recipe( repo, task, roi_id) recipe = Recipe(_recipe) filenames = Filenames('zone', recipe) cog_file = filenames.out_cog_tiff doc_json = Path(filenames.out_cog_tiff + '.geojson') if not doc_json.is_file(): raise OCLIException(f'Could not find "{doc_json.absolute()}"') _json = open(doc_json, 'r').read() try: doc = json.loads(_json) except JSONDecodeError: raise OCLIException(f'Could not parse json "{doc_json.absolute()}"') if not cos_key and "ResultKey" in doc['properties']: cos_key = doc['properties'].get('ResultKey') if not cos_key: raise click.UsageError("No COS key (upload file name)") if not cos_key.endswith('.tiff'): cos_key += '.tiff' log.info( f"About to upload {cog_file} as {cos_key} to bucket {recipe['COS'].get('bucket')} " ) try: cos = COS(recipe) except SystemExit: raise click.UsageError( f'Invalid recipe: COS credentials in "{_recipe}" are required for upload' ) try: if dry_run: output.comment( f'Uploading "{cog_file}" as "{cos_key}" into bucket "{cos.bucket}"' ) output.comment( f'Uploading "{cog_file}.geojson" as "{cos_key}.geojson" into bucket "{cos.bucket}"' ) else: filesize = os.stat(cog_file).st_size with tqdm(total=filesize, unit='B', unit_scale=True, desc=cos_key) as t: cos.upload_to_cos(cog_file, cos_key, hook(t)) if os.path.isfile(cog_file + '.geojson'): filesize = os.stat(cog_file + '.geojson').st_size with tqdm(total=filesize, unit='B', unit_scale=True, desc=cos_key + '.geojson') as t: cos.upload_to_cos(cog_file + '.geojson', cos_key + '.geojson', hook(t)) except SystemExit as e: raise click.UsageError(e)
def ai_preview_stack_math(repo: Repo, task: Task, roi_id, recipe_path, slice_range, show_list, band1, band2, band3, vis_mode, data_path, save, export, hist, ylog): """ Band math for stack {common} """ if not data_path: try: _recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id) recipe = Recipe(_recipe) data_path = recipe.get('DATADIR') output.comment(f'Using recipe file "{recipe_path}"') except (RuntimeError, AssertionError, click.UsageError) as e: output.comment(f'Using stack from task stack_results') try: data_path = task.get_stack_path('snap_path') if not os.path.isdir(data_path): raise AssertionError(f'Directory "{data_path}" is not exists ') except AssertionError as e: raise click.UsageError(f'Could not get stack_results: {e}') output.comment(f"Stack dir: {data_path}\n\n") full_shape, df = get_stack_df(data_path) if show_list: output.table(df, showindex=True) else: title, (r, g, b) = create_stack_rgb(band1, band2, band3, df=df, vis_mode=vis_mode, slice_range=slice_range, ) if export: georef = df.iloc[band1].path _save_envi_rgb(r, g, b, export=export, georef=georef, data_path=data_path, slice_range=slice_range, title=title ) else: _plt = _vis_rgb(r, g, b, title, hist, ylog) _show_plt(_plt, save)
def __init__(self, mode: str, recipe: Recipe, envi: Envi): self.envi = envi self.mode = mode self.recipe = recipe self.envi.DATADIR = self.recipe.get("DATADIR") self.WORKDIR = self.recipe.get("OUTDIR") self.filenames = Filenames(mode, recipe) self.zone = np.array(recipe.get('zone')) self.zone_shape = (self.zone[1][0] - self.zone[0][0], self.zone[1][1] - self.zone[0][1]) self.do_zone = mode == 'zone' self.do_full = mode == 'full' self.tnsr_filename = self.filenames.tnsr
def ai_visualize(repo: Repo, task: Task, roi_id, recipe_path, zone): """visualize AI processing results""" try: _recipe = recipe_path if recipe_path else resolve_recipe( repo, task, roi_id) recipe = Recipe(_recipe) try: cos = COS(recipe) except SystemExit: log.warning("Could not use COS") output.warning("Could not use COS") cos = None envi = Envi(recipe, cos) Visualize(zone, recipe, envi).run() except AssertionError as e: raise click.BadArgumentUsage(str(e))
def ai_preview_cluster(repo: Repo, task: Task, roi_id, recipe_path, slice_range, show_list, band, columns, # threshold, zone, hist, ylog, save, export, rgb=False ): """ Preview assembled tensor band \b * Windows WSL: follow https://www.scivision.dev/pyqtmatplotlib-in-windows-subsystem-for-linux/ """ try: _recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id) recipe = Recipe(_recipe) filenames = Filenames(zone, recipe) pred8c_img = filenames.pred8c_img pred8c_hdr = filenames.pred8c_hdr if not os.path.isfile(pred8c_img): raise AssertionError(f"IMG file '{pred8c_img}' not fond") if not os.path.isfile(pred8c_hdr): raise AssertionError(f"HDR file '{pred8c_hdr}' not fond") pred8c_hdr = s_envi.open(filenames.pred8c_hdr) except (AssertionError) as e: raise click.UsageError(f"Could not visualize: {e}") if show_list: output.comment(f'Cluster HDR: {filenames.pred8c_hdr}') x, y = pred8c_hdr.shape[:2] bn = pred8c_hdr.metadata['band names'] bn = [[b, f'{x}x{y}'] for b in bn] output.table(bn, showindex=True, headers=['band', 'name', 'resolution']) return # if rgb: # if len(rgb) != 3: # raise click.BadOptionUsage('rgb', '--rgb should contain exactly 3 digits without spaces') # band = (int(rgb[0]), int(rgb[1]), int(rgb[2])) if band[0] == -1: band = list(range(0, pred8c_hdr.shape[2])) preview_cluster(filenames.pred8c_hdr, filenames.pred8c_img, band=band, slice_region=slice_range, columns=columns, rgb=rgb )
def ai_predict(repo: Repo, task: Task, roi_id, recipe_path, zone, pred_type): """Run cluster analysis on assembled tensor \b * zone- process full tensor * full- process part of the tensor as defined by "zone" key in a recipe JSON file * fit - run cluster analysis processing only to generate predictor files * fitpredict - run cluster analysis learning and fit * predict - run cluster analysis based on provided in JSON recipe predictor if no --recipe provided, recipe will be taken based on active task """ _recipe = recipe_path if recipe_path else resolve_recipe( repo, task, roi_id) recipe = Recipe(_recipe) with pfac( log, total=100, # show_eta=True, # item_show_func=lambda x: str(x), desc='Processing') as (_, callback): Process(zone, pred_type, recipe).run(callback=callback) pass
def ai_makecog(repo: Repo, task: Task, roi_id, recipe_path, json_only, quiet, no_color, less, zone, kind, source, cos_key, friendly_name, print_res, warp_resampleAlg, overview_resampleAlg): """ Make COG TIFF from visualized results \b * to make GeoTIFF from custom ENVI file use --source option with filename of ENVI file (without extension) * to override recipe kind, use --kind option, example: making image from 'ai preview --export path/to/envi' file use makecog zone --kind Image --source path/to/envi * to avoid overriding recipe main results use --cos-key and --friendly-name option if --friendly-name starts with '+' value will be used as suffix for friendly_name in GeoJSON if --cos-key starts with '+' value will be used as suffix for COS.ResultKey in GeoJSON """ driver = 'MAKECOG' if source: try: # Only valid ENVI or GeoTiff files are alloed str = gdal.Info(source, format='json') driver = str['driverShortName'] if driver not in ['ENVI', 'GTiff']: raise OCLIException( f"Unsupported source file type: {str['driverShortName']} ({str['driverLongName']})" ) except Exception as e: raise OCLIException(f"Option --source: {e}") _recipe = recipe_path if recipe_path else resolve_recipe( repo, task, roi_id) recipe = Recipe(_recipe) kind = kind if kind != 'auto' else recipe.get('type') if kind in ['Rvi', 'Image']: recipe['type'] = 'Image' # log.error(recipe['COS']['ResultKey']) if cos_key: if cos_key.startswith('+'): recipe['COS']['ResultKey'] += cos_key[1:] else: recipe['COS']['ResultKey'] = cos_key if friendly_name: if friendly_name.startswith('+'): recipe['friendly_name'] += friendly_name[1:] else: recipe['friendly_name'] = friendly_name # log.error(recipe['COS']['ResultKey']) # log.error(recipe['friendly_name']) filenames = Filenames(zone, recipe) if source: input_file = source else: input_file = filenames.pred8c_img out_file = filenames.out_tiff cog_file = filenames.out_cog_tiff check_file = cog_file if json_only else input_file if not Path(check_file).is_file(): raise OCLIException(f'file not found: {check_file}') os.makedirs(Path(cog_file).parent, exist_ok=True) w = GDALWrap3(recipe, input_file, out_file, cog_file) try: if not json_only: if driver == 'GTiff': try: shutil.copy(input_file, cog_file) output.success(f"file {input_file} copied to {cog_file}") except Exception as e: raise OCLIException( f'Could not copy "{input_file}" to "{cog_file}" : {e}' ) else: if not quiet: with pfac(log, total=100, desc='Assembling') as (_, callback): def cb(pct, msg, user_data): _ud = user_data[0] # self.log.debug(f"translaing: {round(pct * 100, 2)}% -{msg}- {user_data} {pct}") # reset counter, this t\callback called from multiple prcesses if pct < 0.01: user_data[0] = 0 if user_data[0] == 0 or pct - _ud > 0.10: log.debug( f"Local translating : {round(pct * 100, 2)}%" ) user_data[0] = pct callback(100, pct, 'translating') w.make_cog(cb, warp_resampleAlg, overview_resampleAlg) else: w.make_cog(None, warp_resampleAlg, overview_resampleAlg) _json = w.make_geo_json() _json = _json if no_color else colorful_json(_json) if print_res: click.echo("\n\n\n") if less: click.echo_via_pager(_json) else: click.echo(_json) except (RuntimeError, OCLIException) as e: raise click.UsageError( output.error_style(f"COG-tiff generation failed,reason: {e}")) pass