def pairs_info(repo, all_pairs):
""" show products DB info """
cache_file_name = _cache_pairs_file_name(repo)
_df = pairs.load_from_cache(cache_file_name=cache_file_name)
output.comment("productd DB information\n\n")
rons = _df['relativeOrbitNumber'].unique()
_st = [['Cache file', cache_file_name],
[
'startDate', _df['startDate'].min(axis=0),
_df['startDate'].max(axis=0)
],
[
'completionDate', _df['completionDate'].min(axis=0),
_df['completionDate'].max(axis=0)
], ['Relative orbits', rons],
['Relative orbit number count',
len(rons)], ['records total', len(_df)]]
# output.comment("date intervals\n\n")
output.table(_st, tablefmt='plain')
if all_pairs:
buf = io.StringIO()
_df.info(verbose=True, memory_usage='deep', buf=buf)
_t = buf.getvalue()
output.comment("---------------------")
click.echo(_t)
pass
def wp_info(repo: Repo, list_projects, config):
""" Display information about current tsar install."""
# log.error(f"Here we are {list_projects}")
if list_projects:
_dirs = repo.list_projects()
if not len(_dirs):
comment(f"no projects found in workspace '{repo.projects_home}'")
comment(
" run 'create --name project name' to create new project\n\n")
for v in _dirs:
v['active'] = '*' if v['active'] else ''
if repo.active_project:
output.comment(f'active project "{repo.active_project}"')
else:
output.error(f"No active project selected")
output.table(_dirs,
showindex=True,
headers={
'active': '',
'name': 'name',
'path': 'path'
})
else:
output.comment(
f"config file {os.path.join(repo.rc_home, RC_FILE_NAME)}")
output.comment(
f"Active project config file {repo.get_project_rc_name()}")
output.table(repo.tolist(), headers=['key', 'value'])
def ai_preview_stack(repo: Repo, task: Task, roi_id, recipe_path, slice_range,
show_list,
# rgb,
band, columns, clip, hist, save, export, ylog):
""" Preview assembled tensor band
** use --clip <minl> <max> to apply np.log10(np.clip(.., 10**min, 10**max)) to stack values
\b
* Windows WSL: follow https://www.scivision.dev/pyqtmatplotlib-in-windows-subsystem-for-linux/
"""
try:
_recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id)
recipe = Recipe(_recipe)
_dir = recipe.get('DATADIR')
except (RuntimeError, AssertionError, click.UsageError) as e:
output.comment(f"Could not resolve recipe {e}, fall-back to task")
try:
_dir = task.get_stack_path('snap_path')
except AssertionError as e:
raise click.UsageError(f'Could not get stack path: {e}')
except Exception as e:
log.exception("Could not resolve Stack results")
raise click.UsageError('Could not resolve Stack results')
output.comment(f"Stack dir: {_dir}\n\n")
full_shape, df = get_stack_df(_dir)
if show_list:
output.table(df[['filename', 'resolution', 'path']], showindex=True,
headers=['band', 'name', 'resolution', 'path'])
else:
try:
# if rgb:
# if len(rgb) != 3:
# raise AssertionError('rgb', '--rgb should contain exactly 3 digits without spaces')
# band = (int(rgb[0]), int(rgb[1]), int(rgb[2]))
if band[0] == -1:
band = list(range(0, len(df)))
else:
band = list(band)
_ds = df.iloc[band] # type: gpd.GeoDataFrame
output.table(_ds, showindex=True)
_plt = preview_stack(_ds, _dir,
full_shape=full_shape,
slice_region=slice_range,
band=band,
clip=clip,
columns=columns,
hist=hist,
ylog=ylog
)
_show_plt(_plt, save=save)
except AssertionError as e:
log.exception(e)
raise click.UsageError(str(e))
def _show_tnsr_list(tnsr_hdr_fname, df=None):
output.comment(f'tensor HDR: {tnsr_hdr_fname}')
if df is None:
try:
full_shape, df = get_tensor_df(tnsr_hdr_fname)
except FileNotFoundError as e:
raise OCLIException(f"{e}")
output.table(df, showindex=True, headers=['band', 'name', 'resolution'])
return
def pairs_load(repo: Repo, roi_id, reload, quiet, completion_date):
""" load data into DB """
# todo convert name to ID
if completion_date:
completion_date = parse_to_utc_string(completion_date)
if completion_date is None:
raise OCLIException(
f"Completion date {completion_date} is invalid")
output.comment(f"loading products up to {completion_date}")
if not roi_id and not repo.active_roi:
raise click.BadOptionUsage(
'roi_id',
"ROI is required , set active ROI or provide --roi option")
_id = int(roi_id) if roi_id else int(repo.active_roi)
# check roi exists
db = repo.roi.db
try:
geometry = db.loc[_id, 'geometry']
except KeyError:
raise click.BadOptionUsage('roi_id', f'ROI "{_id}" not found')
cache_file_name = _cache_pairs_file_name(repo)
finder_conf = repo.get_config('finder', {}).copy()
if completion_date:
finder_conf['completionDate'] = completion_date
if not quiet:
output.table(finder_conf.items())
if quiet:
d = pairs.load_data(
geometry,
reload=reload,
callback=None,
finder_conf=finder_conf,
cache_file_name=cache_file_name,
)
else:
with click.progressbar(length=100,
label='Loading sat products') as bar:
def callback(total, step):
if bar.length != total:
bar.length = total
bar.update(step)
d = pairs.load_data(geometry,
reload=reload,
callback=callback,
finder_conf=finder_conf,
cache_file_name=cache_file_name)
if d.empty:
raise OCLIException('0 products loaded, product list is not updated!')
else:
output.success(f'{len(d)} products loaded into list')
def bkt_info(ctx, repo, task: Task, roi_id, bucket_name, reload, less, sort,
limit, column, where, check):
""" show bucket info by BUCKET_NAME OR record number
* list buckets names and record numbers via 'bucket list' command
"""
def_col = [
'productId', 'startDate', 'title', 'relativeOrbitNumber', 'cycle_dt'
]
if column is None or not column:
column = def_col
else:
_cp = []
_cd = []
for c in column:
if c.startswith('+'):
_cp += c[1:].split(',')
else:
_cd.extend(c)
if not _cd:
_cd = def_col
column = _cd + _cp
# log.error(column)
_id, _roi = resolve_roi(roi_id, repo)
if reload:
ctx.invoke(pairs_load, id=_id, reload=True)
bname, _ds, headers = _bkt_info(
repo,
task,
geometry=_roi['geometry'],
bucket_name=bucket_name,
sort=sort,
limit=limit,
column=column,
where=where,
check=check,
)
if limit >= 0:
output.comment(f"Dataset limited to {limit} records")
cols = _ds.columns.to_list()
if task.loaded:
output.comment(f"Task: {task.name}")
if 'task' in cols:
output.comment(
f"INFO: 'task' column: 'm' - used as master in task, 's' - used as slave in task "
)
if 'exists' in cols:
output.comment(
f"INFO: 'exists' column: '+' - full data loaded, '~' - metadata only loaded"
)
output.comment(f'Bucket name: {bname}')
output.table(_ds, headers=headers, less=less)
def parirs_info(repo, value: str, force_id: bool):
""" show products DB info by PRODUCT_ID or RECORD number """
try:
cache_file_name = _cache_pairs_file_name(repo)
_df = pairs.load_from_cache(cache_file_name=cache_file_name)
if value.isnumeric() and not force_id:
_p = _df.iloc[int(value)] # type: Series
else:
_p = _df[_df['productId'] == value].iloc[0] # type: Series
a = [[i, str(v)] for (i, v) in _p.iteritems()]
output.table(a, colalign=('right', ), tablefmt='plain')
except IndexError:
output.error(f'Could not find product by id/record "{value}"')
def ai_preview_cluster(repo: Repo, task: Task, roi_id, recipe_path, slice_range, show_list, band, columns,
# threshold,
zone,
hist, ylog,
save, export,
rgb=False
):
""" Preview assembled tensor band
\b
* Windows WSL: follow https://www.scivision.dev/pyqtmatplotlib-in-windows-subsystem-for-linux/
"""
try:
_recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id)
recipe = Recipe(_recipe)
filenames = Filenames(zone, recipe)
pred8c_img = filenames.pred8c_img
pred8c_hdr = filenames.pred8c_hdr
if not os.path.isfile(pred8c_img):
raise AssertionError(f"IMG file '{pred8c_img}' not fond")
if not os.path.isfile(pred8c_hdr):
raise AssertionError(f"HDR file '{pred8c_hdr}' not fond")
pred8c_hdr = s_envi.open(filenames.pred8c_hdr)
except (AssertionError) as e:
raise click.UsageError(f"Could not visualize: {e}")
if show_list:
output.comment(f'Cluster HDR: {filenames.pred8c_hdr}')
x, y = pred8c_hdr.shape[:2]
bn = pred8c_hdr.metadata['band names']
bn = [[b, f'{x}x{y}'] for b in bn]
output.table(bn, showindex=True, headers=['band', 'name', 'resolution'])
return
# if rgb:
# if len(rgb) != 3:
# raise click.BadOptionUsage('rgb', '--rgb should contain exactly 3 digits without spaces')
# band = (int(rgb[0]), int(rgb[1]), int(rgb[2]))
if band[0] == -1:
band = list(range(0, pred8c_hdr.shape[2]))
preview_cluster(filenames.pred8c_hdr, filenames.pred8c_img,
band=band,
slice_region=slice_range,
columns=columns,
rgb=rgb
)
def pairs_list(repo: Repo, less, path, limit):
""" list directory with SAFE files and its sub-directories and output products and bucket"""
try:
_ds = _list_local(path, limit)
output.table(_ds, less=less, headers='keys')
except ValueError as e:
output.error(f" Bad value: {e}")
except KeyError as e:
output.error(f"column : {e}")
except Exception as e:
log.exception(e)
log.error(f"{traceback.print_tb(e.__traceback__)}")
output.error(
f"No data loaded, run '{pairs_cli.name} {pairs_load.name}' command to initiate cache"
)
def pairs_list(repo: Repo, less, sort, limit, column, where, list_columns,
roi_id, local_only):
""" list sate data """
_id, _roi = resolve_roi(roi_id, repo)
geometry = _roi['geometry']
cache_file_name = _cache_pairs_file_name(repo)
log.debug(f'loading sta data from cache "{cache_file_name}" ')
try:
_df = pairs.load_from_cache(cache_file_name=cache_file_name)
if list_columns:
k = _df.columns.tolist()
output.table([[v] for v in k], headers=['available columns'])
else:
_df = bucket.create_list(_df, buckets_dir='')
_f = list(column)
_ds = _list_products(_df,
where=where,
sort=list(sort),
limit=limit)
_ds['fit'] = _ds['geometry'].intersection(
geometry).area / geometry.area
_f.extend(['fit'])
_ds = _ds[_f]
if local_only:
_locals = [
x['productId'] for x in _list_local(local_only, limit)
]
mask = _ds['productId'].apply(lambda x: x in _locals)
_ds = _ds[mask]
output.table(_ds, headers=['#', *_f], less=less)
return _ds
except ValueError as e:
output.error(f" Bad value: {e}")
except KeyError as e:
output.error(f"column not found : {e}")
except Exception as e:
log.exception(e)
output.error(
f"No data loaded, run '{pairs_cli.name} {pairs_load.name}' command to initiate cache"
)
return None
def ai_preview_stack_math(repo: Repo, task: Task, roi_id, recipe_path, slice_range, show_list,
band1, band2, band3,
vis_mode, data_path, save, export, hist, ylog):
""" Band math for stack
{common}
"""
if not data_path:
try:
_recipe = recipe_path if recipe_path else resolve_recipe(repo, task, roi_id)
recipe = Recipe(_recipe)
data_path = recipe.get('DATADIR')
output.comment(f'Using recipe file "{recipe_path}"')
except (RuntimeError, AssertionError, click.UsageError) as e:
output.comment(f'Using stack from task stack_results')
try:
data_path = task.get_stack_path('snap_path')
if not os.path.isdir(data_path):
raise AssertionError(f'Directory "{data_path}" is not exists ')
except AssertionError as e:
raise click.UsageError(f'Could not get stack_results: {e}')
output.comment(f"Stack dir: {data_path}\n\n")
full_shape, df = get_stack_df(data_path)
if show_list:
output.table(df, showindex=True)
else:
title, (r, g, b) = create_stack_rgb(band1, band2, band3,
df=df,
vis_mode=vis_mode,
slice_range=slice_range,
)
if export:
georef = df.iloc[band1].path
_save_envi_rgb(r, g, b, export=export,
georef=georef, data_path=data_path, slice_range=slice_range,
title=title
)
else:
_plt = _vis_rgb(r, g, b, title, hist, ylog)
_show_plt(_plt, save)
def roi_list(repo: Repo, less):
""" add ROI to project database """
roi = repo.roi
db = roi.db
if not len(db):
raise click.BadOptionUsage(
'project', f"No ROI defined for project '{repo.active_project}'")
try:
cols = list(db.columns)
del cols[cols.index('geometry')]
_ds = db[cols] # double [] to return DataFrame, not Series
mask = db['geometry'].notna()
db = db[mask]
_ds['active'] = None
_db_m = db.to_crs({'init': 'epsg:3857'})
_f = ['minx', 'miny', 'maxx', 'maxy']
_ds['area'] = None
for x in _f:
_ds[x] = None
if not mask.sum():
click.BadParameter('No data')
else:
_ds.loc[mask, _f] = db.loc[mask, 'geometry'].bounds[_f]
_ds.loc[mask, 'area'] = _db_m.loc[mask, 'geometry'].area / 1e6
if repo.active_roi:
_id = int(repo.active_roi)
if _id in db.index:
_ds.iloc[_id]['active'] = '*'
else:
warning('Project active ROI not found in DB')
output.table(_ds,
headers=['ID', *cols, 'active', 'area km2', *_f],
less=less)
click.echo("")
except Exception as e:
log.exception(e)
raise click.BadParameter(f'Unknown error, reason: {e} ')
def bkt_list(ctx: click.Context, repo: Repo, task: Task, roi_id, reload, fit):
_id, _roi = resolve_roi(roi_id, repo)
if reload:
ctx.invoke(pairs_load, roi_id=_id, reload=True)
_m, _s = _task_ms(task)
try:
_t, headers = _bkt_list(
repo,
geometry=_roi['geometry'],
fit=fit,
master=_m,
slave=_s,
)
except RuntimeError as e:
raise click.UsageError(str(e))
output.comment(
f'{len(_t)} possible bucket for roi "{_roi["name"]}" found')
if fit is not None:
_t = _t[_t['fit']['mean'] >= fit]
_l2 = len(_t)
output.comment(f"shown {_l2} with fit >= {fit}")
output.table(_t, headers=headers)
def bkt_info(
repo: Repo,
task: Task,
roi_id,
less,
# sort, limit, column, where,
check,
delta,
product_id,
platform):
""" find pairs by given PRODUCT_ID
\b
PRODUCT_ID: 4-digits hex number (Sentinel product identifier, last 4 symbols in product name).
PLATFORM: like 'S1A' or 'S1B' to narrow search in case PRODUCT_ID is ambiguous
"""
_id, _roi = resolve_roi(roi_id, repo)
_m, _s = _task_ms(task)
geometry = _roi['geometry']
output.comment(f"active task master: {_m}")
_df = pairs.load_from_cache(cache_file_name=(_cache_pairs_file_name(repo)))
_df = _df.set_index('productId')
try:
_ds = _df.loc[product_id][['startDate', 'platform']]
# print( _ds)
if isinstance(_ds, DataFrame):
# print(f"-----{len(_ds)}--------{type(_ds)}----------")
if platform != '':
_ds = _ds[_ds['platform'] == platform].loc[product_id]
if isinstance(_ds, DataFrame):
raise OCLIException(
f"Could not resolve '{product_id}' for platform {platform}"
)
else:
output.table(_ds,
headers=['PRODUCT_ID', 'startDate', 'platform'])
# print( _ds)
raise OCLIException(
f"Product ID {product_id} is ambiguous, use <PALTFORM> argument to narrow search "
)
ts, platform = _ds[['startDate', 'platform']]
# print(f"----------- {ts}")
except KeyError:
raise OCLIException(f'Product id "{product_id}" not found')
output.comment(
f"Building bucket for product {product_id} , startDate={ts}")
f = unitime_delta_factory(ts)
_df['cycle_dt'] = _df['startDate'].apply(f)
_df = _df[(_df['cycle_dt'] <= delta) & (_df['platform'] == platform)]
cols = [
'productId', 'cycle_dt', 'startDate', 'platform',
'relativeOrbitNumber', 'polarisation', 'fit', 'task'
]
try:
if geometry.area == 0:
raise AssertionError('ROI has zero area')
_df['fit'] = _df['geometry'].intersection(
geometry).area / geometry.area
_df['task'] = ''
_df = _df.reset_index()
_df = _df.set_index('title')
if _m in _df.index:
_df.loc[_m, 'task'] = 'm'
else:
output.warning('Current task master not found in bucket')
if _s in _df.index:
_df.loc[_s, 'task'] = 's'
else:
output.warning('Current task slave not found in bucket')
_df = _df.reset_index()
_e, eodata = task.get_valid_key('eodata')
def _ch_fs(b):
_p = _local_eodata_relative_path(eodata, b)
if os.path.isfile(os.path.join(_p, 'manifest.safe')):
_m = os.path.join(_p, 'measurement')
if os.path.isdir(_m):
return '+' if any(os.scandir(_m)) else '~'
return ''
if check and not _e:
_df['exists'] = _df['productIdentifier'].apply(_ch_fs)
cols += ['exists']
pass
_df = _df[cols]
except AssertionError as e:
raise RuntimeError(e)
headers = ['#'] + cols
output.table(
_df,
headers=headers,
)
