def pairs_info(repo, all_pairs): """ show products DB info """ cache_file_name = _cache_pairs_file_name(repo) _df = pairs.load_from_cache(cache_file_name=cache_file_name) output.comment("productd DB information\n\n") rons = _df['relativeOrbitNumber'].unique() _st = [['Cache file', cache_file_name], [ 'startDate', _df['startDate'].min(axis=0), _df['startDate'].max(axis=0) ], [ 'completionDate', _df['completionDate'].min(axis=0), _df['completionDate'].max(axis=0) ], ['Relative orbits', rons], ['Relative orbit number count', len(rons)], ['records total', len(_df)]] # output.comment("date intervals\n\n") output.table(_st, tablefmt='plain') if all_pairs: buf = io.StringIO() _df.info(verbose=True, memory_usage='deep', buf=buf) _t = buf.getvalue() output.comment("---------------------") click.echo(_t) pass
def wp_info(repo: Repo, list_projects, config): """ Display information about current tsar install.""" # log.error(f"Here we are {list_projects}") if list_projects: _dirs = repo.list_projects() if not len(_dirs): comment(f"no projects found in workspace '{repo.projects_home}'") comment( " run 'create --name project name' to create new project\n\n") for v in _dirs: v['active'] = '*' if v['active'] else '' if repo.active_project: output.comment(f'active project "{repo.active_project}"') else: output.error(f"No active project selected") output.table(_dirs, showindex=True, headers={ 'active': '', 'name': 'name', 'path': 'path' }) else: output.comment( f"config file {os.path.join(repo.rc_home, RC_FILE_NAME)}") output.comment( f"Active project config file {repo.get_project_rc_name()}") output.table(repo.tolist(), headers=['key', 'value'])
def ai_preview_stack(repo: Repo, task: Task, roi_id, recipe_path, slice_range, show_list, # rgb, band, columns, clip, hist, save, export, ylog): """ Preview assembled tensor band ** use --clip <minl> <max> to apply np.log10(np.clip(.., 10**min, 10**max)) to stack values \b * Windows WSL: follow https://www.scivision.dev/pyqtmatplotlib-in-windows-subsystem-for-linux/ """ try: _recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id) recipe = Recipe(_recipe) _dir = recipe.get('DATADIR') except (RuntimeError, AssertionError, click.UsageError) as e: output.comment(f"Could not resolve recipe {e}, fall-back to task") try: _dir = task.get_stack_path('snap_path') except AssertionError as e: raise click.UsageError(f'Could not get stack path: {e}') except Exception as e: log.exception("Could not resolve Stack results") raise click.UsageError('Could not resolve Stack results') output.comment(f"Stack dir: {_dir}\n\n") full_shape, df = get_stack_df(_dir) if show_list: output.table(df[['filename', 'resolution', 'path']], showindex=True, headers=['band', 'name', 'resolution', 'path']) else: try: # if rgb: # if len(rgb) != 3: # raise AssertionError('rgb', '--rgb should contain exactly 3 digits without spaces') # band = (int(rgb[0]), int(rgb[1]), int(rgb[2])) if band[0] == -1: band = list(range(0, len(df))) else: band = list(band) _ds = df.iloc[band] # type: gpd.GeoDataFrame output.table(_ds, showindex=True) _plt = preview_stack(_ds, _dir, full_shape=full_shape, slice_region=slice_range, band=band, clip=clip, columns=columns, hist=hist, ylog=ylog ) _show_plt(_plt, save=save) except AssertionError as e: log.exception(e) raise click.UsageError(str(e))
def _show_tnsr_list(tnsr_hdr_fname, df=None): output.comment(f'tensor HDR: {tnsr_hdr_fname}') if df is None: try: full_shape, df = get_tensor_df(tnsr_hdr_fname) except FileNotFoundError as e: raise OCLIException(f"{e}") output.table(df, showindex=True, headers=['band', 'name', 'resolution']) return
def pairs_load(repo: Repo, roi_id, reload, quiet, completion_date): """ load data into DB """ # todo convert name to ID if completion_date: completion_date = parse_to_utc_string(completion_date) if completion_date is None: raise OCLIException( f"Completion date {completion_date} is invalid") output.comment(f"loading products up to {completion_date}") if not roi_id and not repo.active_roi: raise click.BadOptionUsage( 'roi_id', "ROI is required , set active ROI or provide --roi option") _id = int(roi_id) if roi_id else int(repo.active_roi) # check roi exists db = repo.roi.db try: geometry = db.loc[_id, 'geometry'] except KeyError: raise click.BadOptionUsage('roi_id', f'ROI "{_id}" not found') cache_file_name = _cache_pairs_file_name(repo) finder_conf = repo.get_config('finder', {}).copy() if completion_date: finder_conf['completionDate'] = completion_date if not quiet: output.table(finder_conf.items()) if quiet: d = pairs.load_data( geometry, reload=reload, callback=None, finder_conf=finder_conf, cache_file_name=cache_file_name, ) else: with click.progressbar(length=100, label='Loading sat products') as bar: def callback(total, step): if bar.length != total: bar.length = total bar.update(step) d = pairs.load_data(geometry, reload=reload, callback=callback, finder_conf=finder_conf, cache_file_name=cache_file_name) if d.empty: raise OCLIException('0 products loaded, product list is not updated!') else: output.success(f'{len(d)} products loaded into list')
def bkt_info(ctx, repo, task: Task, roi_id, bucket_name, reload, less, sort, limit, column, where, check): """ show bucket info by BUCKET_NAME OR record number * list buckets names and record numbers via 'bucket list' command """ def_col = [ 'productId', 'startDate', 'title', 'relativeOrbitNumber', 'cycle_dt' ] if column is None or not column: column = def_col else: _cp = [] _cd = [] for c in column: if c.startswith('+'): _cp += c[1:].split(',') else: _cd.extend(c) if not _cd: _cd = def_col column = _cd + _cp # log.error(column) _id, _roi = resolve_roi(roi_id, repo) if reload: ctx.invoke(pairs_load, id=_id, reload=True) bname, _ds, headers = _bkt_info( repo, task, geometry=_roi['geometry'], bucket_name=bucket_name, sort=sort, limit=limit, column=column, where=where, check=check, ) if limit >= 0: output.comment(f"Dataset limited to {limit} records") cols = _ds.columns.to_list() if task.loaded: output.comment(f"Task: {task.name}") if 'task' in cols: output.comment( f"INFO: 'task' column: 'm' - used as master in task, 's' - used as slave in task " ) if 'exists' in cols: output.comment( f"INFO: 'exists' column: '+' - full data loaded, '~' - metadata only loaded" ) output.comment(f'Bucket name: {bname}') output.table(_ds, headers=headers, less=less)
def parirs_info(repo, value: str, force_id: bool): """ show products DB info by PRODUCT_ID or RECORD number """ try: cache_file_name = _cache_pairs_file_name(repo) _df = pairs.load_from_cache(cache_file_name=cache_file_name) if value.isnumeric() and not force_id: _p = _df.iloc[int(value)] # type: Series else: _p = _df[_df['productId'] == value].iloc[0] # type: Series a = [[i, str(v)] for (i, v) in _p.iteritems()] output.table(a, colalign=('right', ), tablefmt='plain') except IndexError: output.error(f'Could not find product by id/record "{value}"')
def ai_preview_cluster(repo: Repo, task: Task, roi_id, recipe_path, slice_range, show_list, band, columns, # threshold, zone, hist, ylog, save, export, rgb=False ): """ Preview assembled tensor band \b * Windows WSL: follow https://www.scivision.dev/pyqtmatplotlib-in-windows-subsystem-for-linux/ """ try: _recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id) recipe = Recipe(_recipe) filenames = Filenames(zone, recipe) pred8c_img = filenames.pred8c_img pred8c_hdr = filenames.pred8c_hdr if not os.path.isfile(pred8c_img): raise AssertionError(f"IMG file '{pred8c_img}' not fond") if not os.path.isfile(pred8c_hdr): raise AssertionError(f"HDR file '{pred8c_hdr}' not fond") pred8c_hdr = s_envi.open(filenames.pred8c_hdr) except (AssertionError) as e: raise click.UsageError(f"Could not visualize: {e}") if show_list: output.comment(f'Cluster HDR: {filenames.pred8c_hdr}') x, y = pred8c_hdr.shape[:2] bn = pred8c_hdr.metadata['band names'] bn = [[b, f'{x}x{y}'] for b in bn] output.table(bn, showindex=True, headers=['band', 'name', 'resolution']) return # if rgb: # if len(rgb) != 3: # raise click.BadOptionUsage('rgb', '--rgb should contain exactly 3 digits without spaces') # band = (int(rgb[0]), int(rgb[1]), int(rgb[2])) if band[0] == -1: band = list(range(0, pred8c_hdr.shape[2])) preview_cluster(filenames.pred8c_hdr, filenames.pred8c_img, band=band, slice_region=slice_range, columns=columns, rgb=rgb )
def pairs_list(repo: Repo, less, path, limit): """ list directory with SAFE files and its sub-directories and output products and bucket""" try: _ds = _list_local(path, limit) output.table(_ds, less=less, headers='keys') except ValueError as e: output.error(f" Bad value: {e}") except KeyError as e: output.error(f"column : {e}") except Exception as e: log.exception(e) log.error(f"{traceback.print_tb(e.__traceback__)}") output.error( f"No data loaded, run '{pairs_cli.name} {pairs_load.name}' command to initiate cache" )
def pairs_list(repo: Repo, less, sort, limit, column, where, list_columns, roi_id, local_only): """ list sate data """ _id, _roi = resolve_roi(roi_id, repo) geometry = _roi['geometry'] cache_file_name = _cache_pairs_file_name(repo) log.debug(f'loading sta data from cache "{cache_file_name}" ') try: _df = pairs.load_from_cache(cache_file_name=cache_file_name) if list_columns: k = _df.columns.tolist() output.table([[v] for v in k], headers=['available columns']) else: _df = bucket.create_list(_df, buckets_dir='') _f = list(column) _ds = _list_products(_df, where=where, sort=list(sort), limit=limit) _ds['fit'] = _ds['geometry'].intersection( geometry).area / geometry.area _f.extend(['fit']) _ds = _ds[_f] if local_only: _locals = [ x['productId'] for x in _list_local(local_only, limit) ] mask = _ds['productId'].apply(lambda x: x in _locals) _ds = _ds[mask] output.table(_ds, headers=['#', *_f], less=less) return _ds except ValueError as e: output.error(f" Bad value: {e}") except KeyError as e: output.error(f"column not found : {e}") except Exception as e: log.exception(e) output.error( f"No data loaded, run '{pairs_cli.name} {pairs_load.name}' command to initiate cache" ) return None
def ai_preview_stack_math(repo: Repo, task: Task, roi_id, recipe_path, slice_range, show_list, band1, band2, band3, vis_mode, data_path, save, export, hist, ylog): """ Band math for stack {common} """ if not data_path: try: _recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id) recipe = Recipe(_recipe) data_path = recipe.get('DATADIR') output.comment(f'Using recipe file "{recipe_path}"') except (RuntimeError, AssertionError, click.UsageError) as e: output.comment(f'Using stack from task stack_results') try: data_path = task.get_stack_path('snap_path') if not os.path.isdir(data_path): raise AssertionError(f'Directory "{data_path}" is not exists ') except AssertionError as e: raise click.UsageError(f'Could not get stack_results: {e}') output.comment(f"Stack dir: {data_path}\n\n") full_shape, df = get_stack_df(data_path) if show_list: output.table(df, showindex=True) else: title, (r, g, b) = create_stack_rgb(band1, band2, band3, df=df, vis_mode=vis_mode, slice_range=slice_range, ) if export: georef = df.iloc[band1].path _save_envi_rgb(r, g, b, export=export, georef=georef, data_path=data_path, slice_range=slice_range, title=title ) else: _plt = _vis_rgb(r, g, b, title, hist, ylog) _show_plt(_plt, save)
def roi_list(repo: Repo, less): """ add ROI to project database """ roi = repo.roi db = roi.db if not len(db): raise click.BadOptionUsage( 'project', f"No ROI defined for project '{repo.active_project}'") try: cols = list(db.columns) del cols[cols.index('geometry')] _ds = db[cols] # double [] to return DataFrame, not Series mask = db['geometry'].notna() db = db[mask] _ds['active'] = None _db_m = db.to_crs({'init': 'epsg:3857'}) _f = ['minx', 'miny', 'maxx', 'maxy'] _ds['area'] = None for x in _f: _ds[x] = None if not mask.sum(): click.BadParameter('No data') else: _ds.loc[mask, _f] = db.loc[mask, 'geometry'].bounds[_f] _ds.loc[mask, 'area'] = _db_m.loc[mask, 'geometry'].area / 1e6 if repo.active_roi: _id = int(repo.active_roi) if _id in db.index: _ds.iloc[_id]['active'] = '*' else: warning('Project active ROI not found in DB') output.table(_ds, headers=['ID', *cols, 'active', 'area km2', *_f], less=less) click.echo("") except Exception as e: log.exception(e) raise click.BadParameter(f'Unknown error, reason: {e} ')
def bkt_list(ctx: click.Context, repo: Repo, task: Task, roi_id, reload, fit): _id, _roi = resolve_roi(roi_id, repo) if reload: ctx.invoke(pairs_load, roi_id=_id, reload=True) _m, _s = _task_ms(task) try: _t, headers = _bkt_list( repo, geometry=_roi['geometry'], fit=fit, master=_m, slave=_s, ) except RuntimeError as e: raise click.UsageError(str(e)) output.comment( f'{len(_t)} possible bucket for roi "{_roi["name"]}" found') if fit is not None: _t = _t[_t['fit']['mean'] >= fit] _l2 = len(_t) output.comment(f"shown {_l2} with fit >= {fit}") output.table(_t, headers=headers)
def bkt_info( repo: Repo, task: Task, roi_id, less, # sort, limit, column, where, check, delta, product_id, platform): """ find pairs by given PRODUCT_ID \b PRODUCT_ID: 4-digits hex number (Sentinel product identifier, last 4 symbols in product name). PLATFORM: like 'S1A' or 'S1B' to narrow search in case PRODUCT_ID is ambiguous """ _id, _roi = resolve_roi(roi_id, repo) _m, _s = _task_ms(task) geometry = _roi['geometry'] output.comment(f"active task master: {_m}") _df = pairs.load_from_cache(cache_file_name=(_cache_pairs_file_name(repo))) _df = _df.set_index('productId') try: _ds = _df.loc[product_id][['startDate', 'platform']] # print( _ds) if isinstance(_ds, DataFrame): # print(f"-----{len(_ds)}--------{type(_ds)}----------") if platform != '': _ds = _ds[_ds['platform'] == platform].loc[product_id] if isinstance(_ds, DataFrame): raise OCLIException( f"Could not resolve '{product_id}' for platform {platform}" ) else: output.table(_ds, headers=['PRODUCT_ID', 'startDate', 'platform']) # print( _ds) raise OCLIException( f"Product ID {product_id} is ambiguous, use <PALTFORM> argument to narrow search " ) ts, platform = _ds[['startDate', 'platform']] # print(f"----------- {ts}") except KeyError: raise OCLIException(f'Product id "{product_id}" not found') output.comment( f"Building bucket for product {product_id} , startDate={ts}") f = unitime_delta_factory(ts) _df['cycle_dt'] = _df['startDate'].apply(f) _df = _df[(_df['cycle_dt'] <= delta) & (_df['platform'] == platform)] cols = [ 'productId', 'cycle_dt', 'startDate', 'platform', 'relativeOrbitNumber', 'polarisation', 'fit', 'task' ] try: if geometry.area == 0: raise AssertionError('ROI has zero area') _df['fit'] = _df['geometry'].intersection( geometry).area / geometry.area _df['task'] = '' _df = _df.reset_index() _df = _df.set_index('title') if _m in _df.index: _df.loc[_m, 'task'] = 'm' else: output.warning('Current task master not found in bucket') if _s in _df.index: _df.loc[_s, 'task'] = 's' else: output.warning('Current task slave not found in bucket') _df = _df.reset_index() _e, eodata = task.get_valid_key('eodata') def _ch_fs(b): _p = _local_eodata_relative_path(eodata, b) if os.path.isfile(os.path.join(_p, 'manifest.safe')): _m = os.path.join(_p, 'measurement') if os.path.isdir(_m): return '+' if any(os.scandir(_m)) else '~' return '' if check and not _e: _df['exists'] = _df['productIdentifier'].apply(_ch_fs) cols += ['exists'] pass _df = _df[cols] except AssertionError as e: raise RuntimeError(e) headers = ['#'] + cols output.table( _df, headers=headers, )