def mt_layover(list_of_ls):
import warnings
warnings.filterwarnings('ignore', 'GeoSeries.isna', UserWarning)
target_dir = Path(list_of_ls[0]).parent.parent.parent
bounds = target_dir.joinpath(f'{target_dir.name}.min_bounds.json')
outfile = target_dir.joinpath(f'{target_dir.name}.ls_mask.json')
valid_file = target_dir.joinpath(f'{target_dir.name}.valid.json')
logger.info(
f'Creating common Layover/Shadow mask for track {target_dir.name}.')
for i, file in enumerate(list_of_ls):
if i == 0:
df1 = gpd.read_file(file)
df1 = df1[~(df1.geometry.is_empty | df1.geometry.isna())]
geom = df1.geometry.buffer(0).unary_union
if i > 0:
df2 = gpd.read_file(file)
df2 = df2[~(df2.geometry.is_empty | df2.geometry.isna())]
geom2 = df2.geometry.buffer(0).unary_union
geom = unary_union([geom, geom2])
# make geometry valid in case it isn't
geom = geom.buffer(0)
# remove slivers
buffer = 0.00001
geom = geom.buffer(-buffer, 1,
join_style=2).buffer(buffer,
1,
cap_style=1,
join_style=2).__geo_interface__
# write to output
with open(outfile, 'w') as file:
json.dump(geom, file)
# create difference file for valid data shape
vec.difference(bounds, outfile, valid_file)
def mt_layover_old(list_of_files, config_file):
"""
:param list_of_files:
:param config_file:
:return:
"""
# this is a godale thing
with open(config_file) as file:
config_dict = json.load(file)
temp_dir = Path(config_dict['temp_dir'])
update_extent = (
config_dict['processing']['time-series_ARD']['apply_ls_mask'])
target_dir = Path(list_of_files[0]).parent.parent.parent
outfile = target_dir.joinpath(f'{target_dir.name}.ls_mask.tif')
extent = target_dir.joinpath(f'{target_dir.name}.extent.gpkg')
burst_dir = Path(outfile).parent
burst = burst_dir.name
logger.info(
f'Creating common Layover/Shadow mask for track {target_dir.name}.')
with TemporaryDirectory(prefix=f'{temp_dir}/') as temp:
# temp to Path object
temp = Path(temp)
# create path to temp file
ls_layer = temp.joinpath(Path(outfile).name)
# create a vrt-stack out of
vrt_options = gdal.BuildVRTOptions(srcNodata=0, separate=True)
gdal.BuildVRT(str(temp.joinpath('ls.vrt')),
list_of_files,
options=vrt_options)
with rasterio.open(temp.joinpath('ls.vrt')) as src:
# get metadata
meta = src.meta
# update driver and reduced band count
meta.update(driver='GTiff', count=1, dtype='uint8')
# create outfiles
with rasterio.open(ls_layer, 'w', **meta) as out_min:
# loop through blocks
for _, window in src.block_windows(1):
# read array with all bands
stack = src.read(range(1, src.count + 1), window=window)
# get stats
arr_max = np.nanmax(stack, axis=0)
arr = np.divide(arr_max, arr_max)
out_min.write(np.uint8(arr), window=window, indexes=1)
ras.mask_by_shape(ls_layer,
outfile,
extent,
to_db=False,
datatype='uint8',
rescale=False,
ndv=0)
ls_layer.unlink()
extent_ls_masked = None
if update_extent:
logger.info(
'Calculating symmetrical difference of extent and ls_mask')
# polygonize the multi-temporal ls mask
ras.polygonize_raster(outfile, f'{str(outfile)[:-4]}.gpkg')
# create file for masked extent
extent_ls_masked = burst_dir.joinpath(
f'{burst}.extent.masked.gpkg')
# calculate difference between burst extent
# and ls mask, for masked extent
try:
vec.difference(extent, f'{outfile.stem}.gpkg',
extent_ls_masked)
except:
shutil.copy(extent, extent_ls_masked)
return burst_dir, list_of_files, outfile, extent_ls_masked
def mt_layover(filelist, outfile, temp_dir, extent, update_extent=False):
'''
This function is usally used in the time-series workflow of OST. A list
of the filepaths layover/shadow masks
:param filelist - list of files
:param out_dir - directory where the output file will be stored
:return path to the multi-temporal layover/shadow mask file generated
'''
# get some info
burst_dir = os.path.dirname(outfile)
burst = os.path.basename(burst_dir)
extent = opj(burst_dir, '{}.extent.shp'.format(burst))
# get the start time for Info on processing time
start = time.time()
# create path to out file
ls_layer = opj(temp_dir, os.path.basename(outfile))
# create a vrt-stack out of
print(' INFO: Creating common Layover/Shadow Mask')
vrt_options = gdal.BuildVRTOptions(srcNodata=0, separate=True)
gdal.BuildVRT(opj(temp_dir, 'ls.vrt'), filelist, options=vrt_options)
with rasterio.open(opj(temp_dir, 'ls.vrt')) as src:
# get metadata
meta = src.meta
# update driver and reduced band count
meta.update(driver='GTiff', count=1, dtype='uint8')
# create outfiles
with rasterio.open(ls_layer, 'w', **meta) as out_min:
# loop through blocks
for _, window in src.block_windows(1):
# read array with all bands
stack = src.read(range(1, src.count + 1), window=window)
# get stats
arr_max = np.nanmax(stack, axis=0)
arr = arr_max / arr_max
out_min.write(np.uint8(arr), window=window, indexes=1)
ras.mask_by_shape(ls_layer,
outfile,
extent,
to_db=False,
datatype='uint8',
rescale=False,
ndv=0)
os.remove(ls_layer)
h.timer(start)
if update_extent:
print(' INFO: Calculating symetrical difference of extent and ls_mask')
# polygonize the multi-temporal ls mask
ras.polygonize_raster(outfile, '{}.shp'.format(outfile[:-4]))
# create file for masked extent
extent_ls_masked = opj(burst_dir, '{}.extent.masked.shp'.format(burst))
# calculate difference between burst exntetn and ls mask, fr masked extent
vec.difference(extent, '{}.shp'.format(outfile[:-4]), extent_ls_masked)
def _ard_to_ts(burst_inventory, processing_dir, temp_dir, burst, to_db,
ls_mask_create, ls_mask_apply, mt_speckle_filter, datatype):
burst_dir = opj(processing_dir, burst)
# get common burst extent
list_of_scenes = glob.glob(opj(burst_dir, '20*', '*data*', '*img'))
list_of_scenes = [x for x in list_of_scenes if 'layover' not in x]
extent = opj(burst_dir, '{}.extent.shp'.format(burst))
timeseries.mt_extent(list_of_scenes, extent, buffer=-0.0018)
# remove inital extent
for file in glob.glob(opj(burst_dir, 'tmp*')):
os.remove(file)
# layover/shadow mask
if ls_mask_create is True:
list_of_scenes = glob.glob(opj(burst_dir, '20*', '*data*', '*img'))
list_of_layover = [x for x in list_of_scenes if 'layover' in x]
out_ls = opj(burst_dir, '{}.ls_mask.tif'.format(burst))
timeseries.mt_layover(list_of_layover, out_ls, extent=extent)
logger.debug(
'INFO: Our common layover mask is located at {}'.format(out_ls))
if ls_mask_apply:
logger.debug(
'INFO: Calculating symetrical difference of extent and ls_mask')
ras.polygonize_raster(out_ls, '{}.shp'.format(out_ls[:-4]))
extent_ls_masked = opj(burst_dir, '{}.extent.masked.shp'.format(burst))
vec.difference(extent, '{}.shp'.format(out_ls[:-4]), extent_ls_masked)
extent = extent_ls_masked
list_of_product_types = {'BS': 'Gamma0', 'coh': 'coh', 'ha_alpha': 'Alpha'}
# we loop through each possible product
for p, product_name in list_of_product_types.items():
# we loop through each polarisation
for pol in ['VV', 'VH', 'HH', 'HV']:
# see if there is actually any imagery
list_of_ts_bursts = sorted(
glob.glob(
opj(processing_dir, burst, '20*', '*data*',
'{}*{}*img'.format(product_name, pol))))
if len(list_of_ts_bursts) > 1:
# check for all datafiles of this product type
list_of_ts_bursts = sorted(
glob.glob(
opj(processing_dir, burst, '20*/',
'*{}*dim'.format(p))))
list_of_ts_bursts = '\'{}\''.format(
','.join(list_of_ts_bursts))
# define out_dir for stacking routine
out_dir = opj(processing_dir, '{}/Timeseries'.format(burst))
os.makedirs(out_dir, exist_ok=True)
temp_stack = opj(temp_dir, '{}_{}_{}_mt'.format(burst, p, pol))
out_stack = opj(out_dir, '{}_{}_{}_mt'.format(burst, p, pol))
stack_log = opj(out_dir,
'{}_{}_{}_stack.err_log'.format(burst, p, pol))
# run stacking routines
ts.create_stack(list_of_ts_bursts,
temp_stack,
stack_log,
polarisation=pol)
# run mt speckle filter
if mt_speckle_filter is True:
speckle_log = opj(
out_dir,
'{}_{}_{}_mt_speckle.err_log'.format(burst, p, pol))
ts.mt_speckle_filter('{}.dim'.format(temp_stack),
out_stack, speckle_log)
# remove tmp files
h.delete_dimap(temp_stack)
else:
out_stack = temp_stack
# convert to GeoTiffs
if p == 'BS':
# get the dates of the files
dates = [
datetime.datetime.strptime(
x.split('_')[-1][:-4], '%d%b%Y')
for x in glob.glob(
opj('{}.data'.format(out_stack), '*img'))
]
# sort them
dates.sort()
# write them back to string for following loop
sorted_dates = [
datetime.datetime.strftime(ts, "%d%b%Y")
for ts in dates
]
i, outfiles = 1, []
for date in sorted_dates:
# restructure date to YYMMDD
in_date = datetime.datetime.strptime(date, '%d%b%Y')
out_date = datetime.datetime.strftime(
in_date, '%y%m%d')
infile = glob.glob(
opj('{}.data'.format(out_stack),
'*{}*{}*img'.format(pol, date)))[0]
# create outfile
outfile = opj(
out_dir,
'{}.{}.{}.{}.tif'.format(i, out_date, p, pol))
# mask by extent
ras.to_gtiff_clip_by_extend(infile,
outfile,
extent,
to_db=to_db,
out_dtype=datatype,
min_value=-30,
max_value=5,
no_data=0.0)
# add ot a list for subsequent vrt creation
outfiles.append(outfile)
i += 1
# build vrt of timeseries
vrt_options = gdal.BuildVRTOptions(srcNodata=0,
separate=True)
gdal.BuildVRT(opj(out_dir,
'Timeseries.{}.{}.vrt'.format(p, pol)),
outfiles,
options=vrt_options)
if p == 'coh':
# get slave and master Date
mstDates = [
datetime.datetime.strptime(
os.path.basename(x).split('_')[3].split('.')[0],
'%d%b%Y') for x in glob.glob(
opj('{}.data'.format(out_stack), '*img'))
]
slvDates = [
datetime.datetime.strptime(
os.path.basename(x).split('_')[4].split('.')[0],
'%d%b%Y') for x in glob.glob(
opj('{}.data'.format(out_stack), '*img'))
]
# sort them
mstDates.sort()
slvDates.sort()
# write them back to string for following loop
sortedMstDates = [
datetime.datetime.strftime(ts, "%d%b%Y")
for ts in mstDates
]
sortedSlvDates = [
datetime.datetime.strftime(ts, "%d%b%Y")
for ts in slvDates
]
i, outfiles = 1, []
for mst, slv in zip(sortedMstDates, sortedSlvDates):
inMst = datetime.datetime.strptime(mst, '%d%b%Y')
inSlv = datetime.datetime.strptime(slv, '%d%b%Y')
outMst = datetime.datetime.strftime(inMst, '%y%m%d')
outSlv = datetime.datetime.strftime(inSlv, '%y%m%d')
infile = glob.glob(
opj('{}.data'.format(out_stack),
'*{}*{}_{}*img'.format(pol, mst, slv)))[0]
outfile = opj(
out_dir, '{}.{}.{}.{}.{}.tif'.format(
i, outMst, outSlv, p, pol))
ras.to_gtiff_clip_by_extend(infile,
outfile,
extent,
to_db=False,
out_dtype=datatype,
min_value=0.000001,
max_value=1,
no_data=0.0)
# add ot a list for subsequent vrt creation
outfiles.append(outfile)
i += 1
# build vrt of timeseries
vrt_options = gdal.BuildVRTOptions(srcNodata=0,
separate=True)
gdal.BuildVRT(opj(out_dir,
'Timeseries.{}.{}.vrt'.format(p, pol)),
outfiles,
options=vrt_options)
# remove tmp files
h.delete_dimap(out_stack)
for pol in ['Alpha', 'Entropy', 'Anisotropy']:
list_of_ts_bursts = sorted(
glob.glob(
opj(processing_dir, burst, '20*', '*{}*'.format(p),
'*{}.img'.format(pol))))
if len(list_of_ts_bursts) > 1:
list_of_ts_bursts = sorted(
glob.glob(
opj(processing_dir, burst, '20*/', '*{}*dim'.format(p))))
list_of_ts_bursts = '\'{}\''.format(','.join(list_of_ts_bursts))
# logger.debug(list_of_ts_bursts)
out_dir = opj(processing_dir, '{}/Timeseries'.format(burst))
os.makedirs(out_dir, exist_ok=True)
temp_stack = opj(temp_dir, '{}_{}_mt'.format(burst, pol))
out_stack = opj(out_dir, '{}_{}_mt'.format(burst, pol))
stack_log = opj(out_dir, '{}_{}_stack.err_log'.format(burst, pol))
# processing routines
ts.create_stack(list_of_ts_bursts,
temp_stack,
stack_log,
pattern=pol)
if mt_speckle_filter is True:
speckle_log = opj(
out_dir, '{}_{}_mt_speckle.err_log'.format(burst, pol))
ts.mt_speckle_filter('{}.dim'.format(temp_stack), out_stack,
speckle_log)
# remove tmp files
h.delete_dimap(temp_stack)
else:
out_stack = temp_stack
# get the dates of the files
dates = [
datetime.datetime.strptime(x.split('_')[-1][:-4], '%d%b%Y')
for x in glob.glob(opj('{}.data'.format(out_stack), '*img'))
]
# sort them
dates.sort()
# write them back to string for following loop
sorted_dates = [
datetime.datetime.strftime(ts, "%d%b%Y") for ts in dates
]
i, outfiles = 1, []
for date in sorted_dates:
# restructure date to YYMMDD
in_date = datetime.datetime.strptime(date, '%d%b%Y')
out_date = datetime.datetime.strftime(in_date, '%y%m%d')
infile = glob.glob(
opj('{}.data'.format(out_stack),
'*{}*{}*img'.format(pol, date)))[0]
# create outfile
outfile = opj(out_dir,
'{}.{}.{}.{}.tif'.format(i, out_date, p, pol))
# mask by extent
max_value = 90 if pol is 'Alpha' else 1
ras.to_gtiff_clip_by_extend(infile,
outfile,
extent,
to_db=False,
out_dtype=datatype,
min_value=0.000001,
max_value=max_value,
no_data=0)
# add ot a list for subsequent vrt creation
outfiles.append(outfile)
i += 1
# build vrt of timeseries
vrt_options = gdal.BuildVRTOptions(srcNodata=0, separate=True)
gdal.BuildVRT(opj(out_dir, 'Timeseries.{}.vrt'.format(pol)),
outfiles,
options=vrt_options)
# remove tmp files
h.delete_dimap(out_stack)