def mt_metrics(stack, out_prefix, metrics, rescale_to_datatype, to_power,
outlier_removal, datelist):
"""
:param stack:
:param out_prefix:
:param metrics:
:param rescale_to_datatype:
:param to_power:
:param outlier_removal:
:param datelist:
:return:
"""
logger.info(f'Creating timescan layers ({metrics}) of track/burst '
f'{out_prefix.parent.parent.name} for {out_prefix.name}')
warnings.filterwarnings('ignore', r'All-NaN (slice|axis) encountered')
warnings.filterwarnings('ignore', r'Mean of empty slice')
warnings.filterwarnings('ignore', r'Degrees of freedom', RuntimeWarning)
harmonics = False
if 'harmonics' in metrics:
logger.info('Calculating harmonics')
if not datelist:
raise RuntimeWarning('Harmonics need the datelist. '
'Harmonics will not be calculated')
else:
harmonics = True
metrics.remove('harmonics')
metrics.extend(
['amplitude', 'phase', 'residuals', 'trend', 'model_mean'])
if 'percentiles' in metrics:
metrics.remove('percentiles')
metrics.extend(['p95', 'p5'])
with rasterio.open(stack) as src:
# get metadata
meta = src.profile
# update driver and reduced band count
meta.update({'driver': 'GTiff'})
meta.update({'count': 1})
# write all different output files into a dictionary
metric_dict = {}
for metric in metrics:
filename = f'{out_prefix}.{metric}.tif'
metric_dict[metric] = rasterio.open(filename, 'w', **meta)
# scaling factors in case we have to rescale to integer
minimums = {
'avg': int(-30),
'max': int(-30),
'min': int(-30),
'median': -30,
'p5': -30,
'p95': -30,
'std': 0.00001,
'cov': 0.00001,
'amplitude': -5,
'phase': -np.pi,
'residuals': -10,
'trend': -5,
'model_mean': -30
}
maximums = {
'avg': 5,
'max': 5,
'min': 5,
'median': 5,
'p5': 5,
'p95': 5,
'std': 0.2,
'cov': 1,
'amplitude': 5,
'phase': np.pi,
'residuals': 10,
'trend': 5,
'model_mean': 5
}
if 'amplitude' in metrics:
# construct independent variables
dates, sines, cosines, intercept = [], [], [], []
two_pi = np.multiply(2, np.pi)
for date in sorted(datelist):
delta = difference_in_years(
datetime.strptime('700101', "%y%m%d"),
datetime.strptime(date, "%y%m%d"))
dates.append(delta)
sines.append(np.sin(np.multiply(two_pi, delta)))
cosines.append(np.cos(np.multiply(two_pi, delta)))
intercept.append(1)
x_array = np.array([dates, cosines, sines, intercept])
# loop through blocks
for _, window in src.block_windows(1):
# read array with all bands
stack = src.read(range(1, src.count + 1), window=window)
# rescale to float
if rescale_to_datatype is True and meta['dtype'] != 'float32':
stack = ras.rescale_to_float(stack, meta['dtype'])
# transform to power
if to_power is True:
stack = np.power(10, np.divide(stack, 10))
# outlier removal (only applies if there are more than 5 bands)
if outlier_removal is True and src.count >= 5:
stack = remove_outliers(stack)
# get stats
arr = {
'p95': (nan_percentile(stack, [95, 5]) if 'p95' in metrics else
(False, False))[0],
'p5': (nan_percentile(stack, [95, 5]) if 'p95' in metrics else
(False, False))[1],
'median': (np.nanmedian(stack, axis=0)
if 'median' in metrics else False),
'avg':
(np.nanmean(stack, axis=0) if 'avg' in metrics else False),
'max':
(np.nanmax(stack, axis=0) if 'max' in metrics else False),
'min':
(np.nanmin(stack, axis=0) if 'min' in metrics else False),
'std':
(np.nanstd(stack, axis=0) if 'std' in metrics else False),
#'cov': (stats.variation(stack, axis=0, nan_policy='omit')
'cov':
(np.divide(np.nanstd(stack, axis=0), np.nanmean(stack, axis=0))
if 'cov' in metrics else False)
}
if 'amplitude' in metrics:
stack_size = (stack.shape[1], stack.shape[2])
if to_power is True:
y = ras.convert_to_db(stack).reshape(stack.shape[0], -1)
else:
y = stack.reshape(stack.shape[0], -1)
x, residuals, _, _ = np.linalg.lstsq(x_array.T, y, rcond=-1)
arr['amplitude'] = np.hypot(x[1], x[2]).reshape(stack_size)
arr['phase'] = np.arctan2(x[2], x[1]).reshape(stack_size)
arr['trend'] = x[0].reshape(stack_size)
arr['model_mean'] = x[3].reshape(stack_size)
arr['residuals'] = np.sqrt(np.divide(
residuals, stack.shape[0])).reshape(stack_size)
# the metrics to be re-turned to dB, in case to_power is True
metrics_to_convert = ['avg', 'min', 'max', 'p95', 'p5', 'median']
# do the back conversions and write to disk loop
for metric in metrics:
if to_power is True and metric in metrics_to_convert:
arr[metric] = ras.convert_to_db(arr[metric])
if ((rescale_to_datatype is True
and meta['dtype'] != 'float32')
or (metric in ['cov', 'phase']
and meta['dtype'] != 'float32')):
arr[metric] = ras.scale_to_int(arr[metric],
minimums[metric],
maximums[metric],
meta['dtype'])
# write to dest
metric_dict[metric].write(np.nan_to_num(arr[metric]).astype(
meta['dtype']),
window=window,
indexes=1)
metric_dict[metric].update_tags(
1, BAND_NAME=f'{Path(out_prefix).name}_{metric}')
metric_dict[metric].set_band_description(
1, f'{Path(out_prefix).name}_{metric}')
# close the output files
for metric in metrics:
# close rio opening
metric_dict[metric].close()
# construct filename
filename = f'{str(out_prefix)}.{metric}.tif'
return_code = h.check_out_tiff(filename)
if return_code != 0:
for metric_ in metrics:
# remove all files and return
filename = f'{str(out_prefix)}.{metric_}.tif'
Path(filename).unlink()
if Path(f'{filename}.xml').exists():
Path(f'{filename}.xml').unlink()
return None, None, None, return_code
# write out that it's been processed
dirname = out_prefix.parent
check_file = dirname.joinpath(f'.{out_prefix.name}.processed')
with open(str(check_file), 'w') as file:
file.write('passed all tests \n')
target = out_prefix.parent.parent.name
return target, out_prefix.name, metrics, None
def ard_to_ts(list_of_files, burst, product, pol, config_file):
# -------------------------------------------
# 1 unpack list of args
# convert list of files readable for snap
list_of_files = f"\'{','.join(str(x) for x in list_of_files)}\'"
# -------------------------------------------
# 2 read config file
with open(config_file, 'r') as file:
config_dict = json.load(file)
processing_dir = Path(config_dict['processing_dir'])
ard = config_dict['processing']['single_ARD']
ard_mt = config_dict['processing']['time-series_ARD']
# -------------------------------------------
# 3 get namespace of directories and check if already processed
# get the burst directory
burst_dir = processing_dir.joinpath(burst)
# get timeseries directory and create if non existent
out_dir = burst_dir.joinpath('Timeseries')
Path.mkdir(out_dir, parents=True, exist_ok=True)
# in case some processing has been done before, check if already processed
check_file = out_dir.joinpath(f'.{product}.{pol}.processed')
if Path.exists(check_file):
logger.info(
f'Timeseries of {burst} for {product} in {pol} '
f'polarisation already processed.'
)
out_files = 'already_processed'
out_vrt = 'already_processed'
return (
burst, list_of_files, out_files, out_vrt, f'{product}.{pol}', None
)
# -------------------------------------------
# 4 adjust processing parameters according to config
# get the db scaling right
to_db = ard['to_db']
if to_db or product != 'bs':
to_db = False
logger.debug(f'Not converting to dB for {product}')
else:
to_db = ard_mt['to_db']
logger.debug(f'Converting to dB for {product}')
if ard_mt['apply_ls_mask']:
extent = burst_dir.joinpath(f'{burst}.valid.json')
else:
extent = burst_dir.joinpath(f'{burst}.min_bounds.json')
# -------------------------------------------
# 5 SNAP processing
with TemporaryDirectory(prefix=f"{config_dict['temp_dir']}/") as temp:
# turn to Path object
temp = Path(temp)
# create namespaces
temp_stack = temp.joinpath(f'{burst}_{product}_{pol}')
out_stack = temp.joinpath(f'{burst}_{product}_{pol}_mt')
stack_log = out_dir.joinpath(f'{burst}_{product}_{pol}_stack.err_log')
# run stacking routine
if pol in ['Alpha', 'Anisotropy', 'Entropy']:
logger.info(
f'Creating multi-temporal stack of images of burst/track '
f'{burst} for the {pol} band of the polarimetric '
f'H-A-Alpha decomposition.'
)
try:
create_stack(
list_of_files, temp_stack, stack_log, config_dict,
pattern=pol
)
except (GPTRuntimeError, NotValidFileError) as error:
logger.info(error)
return None, None, None, None, None, error
else:
logger.info(
f'Creating multi-temporal stack of images of burst/track '
f'{burst} for {product} product in {pol} polarization.'
)
try:
create_stack(
list_of_files, temp_stack, stack_log, config_dict,
polarisation=pol
)
except (GPTRuntimeError, NotValidFileError) as error:
logger.info(error)
return None, None, None, None, None, error
# run mt speckle filter
if ard_mt['remove_mt_speckle'] is True:
speckle_log = out_dir.joinpath(
f'{burst}_{product}_{pol}_mt_speckle.err_log'
)
logger.debug('Applying multi-temporal speckle filter')
try:
mt_speckle_filter(
temp_stack.with_suffix('.dim'), out_stack, speckle_log,
config_dict
)
except (GPTRuntimeError, NotValidFileError) as error:
logger.info(error)
return None, None, None, None, None, error
# remove tmp files
h.delete_dimap(temp_stack)
else:
out_stack = temp_stack
# -----------------------------------------------
# 6 Conversion to GeoTiff
# min max dict for stretching in case of 16 or 8 bit datatype
mm_dict = {'bs': {'min': -30, 'max': 5},
'coh': {'min': 0.000001, 'max': 1},
'Alpha': {'min': 0.000001, 'max': 90},
'Anisotropy': {'min': 0.000001, 'max': 1},
'Entropy': {'min': 0.000001, 'max': 1}
}
stretch = pol if pol in ['Alpha', 'Anisotropy', 'Entropy'] else product
if product == 'coh':
# get slave and master dates from file names and sort them
mst_dates = sorted([
dt.strptime(
file.name.split('_')[3].split('.')[0], SNAP_DATEFORMAT
)
for file in list(out_stack.with_suffix('.data').glob('*.img'))
])
slv_dates = sorted([
dt.strptime(
file.name.split('_')[4].split('.')[0], SNAP_DATEFORMAT
)
for file in list(out_stack.with_suffix('.data').glob('*.img'))
])
# write them back to string for following loop
mst_dates = [dt.strftime(ts, SNAP_DATEFORMAT) for ts in mst_dates]
slv_dates = [dt.strftime(ts, SNAP_DATEFORMAT) for ts in slv_dates]
out_files = []
for i, (mst, slv) in enumerate(zip(mst_dates, slv_dates)):
# re-construct namespace for input file
infile = list(
out_stack.with_suffix('.data').glob(
f'*{pol}*{mst}_{slv}*img'
)
)[0]
# rename dates to YYYYMMDD format
mst = dt.strftime(dt.strptime(mst, SNAP_DATEFORMAT), '%y%m%d')
slv = dt.strftime(dt.strptime(slv, SNAP_DATEFORMAT), '%y%m%d')
# create namespace for output file with renamed dates
outfile = out_dir.joinpath(
f'{i+1:02d}.{mst}.{slv}.{product}.{pol}.tif'
)
# fill internal values if any
#with rasterio.open(str(infile), 'r') as src:
# meta = src.meta.copy()
# filled = ras.fill_internal_nans(src.read())
#with rasterio.open(str(infile), 'w', **meta) as dest:
# dest.write(filled)
#print('filled')
# produce final outputfile,
# including dtype conversion and ls mask
ras.mask_by_shape(
infile, outfile, extent, to_db=to_db,
datatype=ard_mt['dtype_output'],
min_value=mm_dict[stretch]['min'],
max_value=mm_dict[stretch]['max'],
ndv=0.0, description=True)
# add ot a list for subsequent vrt creation
out_files.append(str(outfile))
else:
# get the dates of the files
dates = sorted([dt.strptime(
file.name.split('_')[-1][:-4], SNAP_DATEFORMAT)
for file in list(out_stack.with_suffix('.data').glob('*.img'))
])
# write them back to string for following loop
dates = [dt.strftime(ts, "%d%b%Y") for ts in dates]
out_files = []
for i, date in enumerate(dates):
# re-construct namespace for input file
infile = list(
out_stack.with_suffix('.data').glob(f'*{pol}*{date}*img')
)[0]
# restructure date to YYMMDD
date = dt.strftime(
dt.strptime(date, SNAP_DATEFORMAT), '%y%m%d'
)
# create namespace for output file
outfile = out_dir.joinpath(
f'{i+1:02d}.{date}.{product}.{pol}.tif'
)
# fill internal nodata
#if ard['image_type'] == 'SLC':
#with rasterio.open(str(infile), 'r') as src:
# meta = src.meta.copy()
#filled = ras.fill_internal_nans(src.read())
#with rasterio.open(str(infile), 'w', **meta) as dest:
# dest.write(filled)
#print('filledbs')
# run conversion routine
ras.mask_by_shape(infile, outfile, extent,
to_db=to_db,
datatype=ard_mt['dtype_output'],
min_value=mm_dict[stretch]['min'],
max_value=mm_dict[stretch]['max'],
ndv=0.0)
# add ot a list for subsequent vrt creation
out_files.append(str(outfile))
# -----------------------------------------------
# 7 Filechecks
for file in out_files:
return_code = h.check_out_tiff(file)
if return_code != 0:
for file_ in out_files:
Path(file_).unlink()
if Path(f'{file}.xml').exists():
Path(f'{file}.xml').unlink()
return (
burst, list_of_files, None, None,
f'{product}.{pol}', return_code
)
# write file, so we know this ts has been successfully processed
with open(str(check_file), 'w') as file:
file.write('passed all tests \n')
# -----------------------------------------------
# 8 Create vrts
vrt_options = gdal.BuildVRTOptions(srcNodata=0, separate=True)
out_vrt = str(out_dir.joinpath(f'Timeseries.{product}.{pol}.vrt'))
gdal.BuildVRT(
out_vrt,
out_files,
options=vrt_options
)
return burst, list_of_files, out_files, out_vrt, f'{product}.{pol}', None
def mosaic(filelist, outfile, temp_dir, cut_to_aoi=False):
check_file = opj(os.path.dirname(outfile),
'.{}.processed'.format(os.path.basename(outfile)[:-4]))
logfile = opj(os.path.dirname(outfile),
'{}.errLog'.format(os.path.basename(outfile)[:-4]))
with rasterio.open(filelist.split(' ')[0]) as src:
dtype = src.meta['dtype']
dtype = 'float' if dtype == 'float32' else dtype
if cut_to_aoi:
tempfile = opj(temp_dir, os.path.basename(outfile))
else:
tempfile = outfile
cmd = ('otbcli_Mosaic -ram 4096'
' -progress 1'
' -comp.feather large'
' -harmo.method band'
' -harmo.cost rmse'
' -temp_dir {}'
' -il {}'
' -out {} {}'.format(temp_dir, filelist, tempfile, dtype))
return_code = h.run_command(cmd, logfile)
if return_code != 0:
if os.path.isfile(tempfile):
os.remove(tempfile)
return
if cut_to_aoi:
# get aoi ina way rasterio wants it
features = vec.gdf_to_json_geometry(vec.wkt_to_gdf(cut_to_aoi))
# import raster and mask
with rasterio.open(tempfile) as src:
out_image, out_transform = rasterio.mask.mask(src,
features,
crop=True)
out_meta = src.meta.copy()
ndv = src.nodata
out_image = np.ma.masked_where(out_image == ndv, out_image)
out_meta.update({
'driver': 'GTiff',
'height': out_image.shape[1],
'width': out_image.shape[2],
'transform': out_transform,
'tiled': True,
'blockxsize': 128,
'blockysize': 128
})
with rasterio.open(outfile, 'w', **out_meta) as dest:
dest.write(out_image.data)
# remove intermediate file
os.remove(tempfile)
# check
return_code = h.check_out_tiff(outfile)
if return_code != 0:
if os.path.isfile(outfile):
os.remove(outfile)
# write file, so we know this ts has been succesfully processed
if return_code == 0:
with open(str(check_file), 'w') as file:
file.write('passed all tests \n')
def ard_to_ts(list_of_files,
processing_dir,
temp_dir,
burst,
proc_file,
product,
pol,
ncores=os.cpu_count()):
if type(list_of_files) == str:
list_of_files = list_of_files.replace("'", '').strip('][').split(', ')
# get the burst directory
burst_dir = opj(processing_dir, burst)
# check routine if timeseries has already been processed
check_file = opj(burst_dir, 'Timeseries',
'.{}.{}.processed'.format(product, pol))
if os.path.isfile(check_file):
print(' INFO: Timeseries of {} for {} in {} polarisation already'
' processed'.format(burst, product, pol))
return
# load ard parameters
with open(proc_file, 'r') as ard_file:
ard_params = json.load(ard_file)['processing parameters']
ard = ard_params['single ARD']
ard_mt = ard_params['time-series ARD']
if ard_mt['remove mt speckle'] is True:
ard_mt_speck = ard_params['time-series ARD']['mt speckle filter']
# get the db scaling right
to_db = ard['to db']
if to_db or product != 'bs':
to_db = False
print('INFO: Not converting to dB for {}'.format(product))
else:
to_db = ard_mt['to db']
print('INFO: Converting to dB for {}'.format(product))
if ard['apply ls mask']:
extent = opj(burst_dir, '{}.extent.masked.shp'.format(burst))
else:
extent = opj(burst_dir, '{}.extent.shp'.format(burst))
# min max dict for stretching in case of 16 or 8 bit datatype
mm_dict = {
'bs': {
'min': -30,
'max': 5
},
'coh': {
'min': 0.000001,
'max': 1
},
'Alpha': {
'min': 0.000001,
'max': 90
},
'Anisotropy': {
'min': 0.000001,
'max': 1
},
'Entropy': {
'min': 0.000001,
'max': 1
}
}
stretch = pol if pol in ['Alpha', 'Anisotropy', 'Entropy'] else product
# define out_dir for stacking routine
out_dir = opj(processing_dir, '{}'.format(burst), 'Timeseries')
os.makedirs(out_dir, exist_ok=True)
# create namespaces
temp_stack = opj(temp_dir, '{}_{}_{}'.format(burst, product, pol))
out_stack = opj(temp_dir, '{}_{}_{}_mt'.format(burst, product, pol))
stack_log = opj(out_dir,
'{}_{}_{}_stack.err_log'.format(burst, product, pol))
# run stacking routines
# convert list of files readable for snap
list_of_files = '\'{}\''.format(','.join(list_of_files))
if pol in ['Alpha', 'Anisotropy', 'Entropy']:
print(
' INFO: Creating multi-temporal stack of images of burst/track {} for'
' the {} band of the polarimetric H-A-Alpha'
' decomposition.'.format(burst, pol))
create_stack(list_of_files, temp_stack, stack_log, pattern=pol)
else:
print(
' INFO: Creating multi-temporal stack of images of burst/track {} for'
' {} product in {} polarization.'.format(burst, product, pol))
create_stack(list_of_files, temp_stack, stack_log, polarisation=pol)
# run mt speckle filter
if ard_mt['remove mt speckle'] is True:
speckle_log = opj(
out_dir, '{}_{}_{}_mt_speckle.err_log'.format(burst, product, pol))
print(' INFO: Applying multi-temporal speckle filter')
mt_speckle_filter('{}.dim'.format(temp_stack),
out_stack,
speckle_log,
speckle_dict=ard_mt_speck,
ncores=ncores)
# remove tmp files
h.delete_dimap(temp_stack)
else:
out_stack = temp_stack
if product == 'coh':
# get slave and master Date
mstDates = [
datetime.datetime.strptime(
os.path.basename(x).split('_')[3].split('.')[0], '%d%b%Y')
for x in glob.glob(opj('{}.data'.format(out_stack), '*img'))
]
slvDates = [
datetime.datetime.strptime(
os.path.basename(x).split('_')[4].split('.')[0], '%d%b%Y')
for x in glob.glob(opj('{}.data'.format(out_stack), '*img'))
]
# sort them
mstDates.sort()
slvDates.sort()
# write them back to string for following loop
sortedMstDates = [
datetime.datetime.strftime(ts, "%d%b%Y") for ts in mstDates
]
sortedSlvDates = [
datetime.datetime.strftime(ts, "%d%b%Y") for ts in slvDates
]
i, outfiles = 1, []
for mst, slv in zip(sortedMstDates, sortedSlvDates):
inMst = datetime.datetime.strptime(mst, '%d%b%Y')
inSlv = datetime.datetime.strptime(slv, '%d%b%Y')
outMst = datetime.datetime.strftime(inMst, '%y%m%d')
outSlv = datetime.datetime.strftime(inSlv, '%y%m%d')
infile = glob.glob(
opj('{}.data'.format(out_stack),
'*{}*{}_{}*img'.format(pol, mst, slv)))[0]
outfile = opj(
out_dir,
'{:02d}.{}.{}.{}.{}.tif'.format(i, outMst, outSlv, product,
pol))
ras.mask_by_shape(infile,
outfile,
extent,
to_db=to_db,
datatype=ard_mt['dtype output'],
min_value=mm_dict[stretch]['min'],
max_value=mm_dict[stretch]['max'],
ndv=0.0,
description=True)
# add ot a list for subsequent vrt creation
outfiles.append(outfile)
i += 1
else:
# get the dates of the files
dates = [
datetime.datetime.strptime(x.split('_')[-1][:-4], '%d%b%Y')
for x in glob.glob(opj('{}.data'.format(out_stack), '*img'))
]
# sort them
dates.sort()
# write them back to string for following loop
sortedDates = [
datetime.datetime.strftime(ts, "%d%b%Y") for ts in dates
]
i, outfiles = 1, []
for date in sortedDates:
# restructure date to YYMMDD
inDate = datetime.datetime.strptime(date, '%d%b%Y')
outDate = datetime.datetime.strftime(inDate, '%y%m%d')
infile = glob.glob(
opj('{}.data'.format(out_stack),
'*{}*{}*img'.format(pol, date)))[0]
# create outfile
outfile = opj(
out_dir, '{:02d}.{}.{}.{}.tif'.format(i, outDate, product,
pol))
ras.mask_by_shape(infile,
outfile,
extent,
to_db=to_db,
datatype=ard_mt['dtype output'],
min_value=mm_dict[stretch]['min'],
max_value=mm_dict[stretch]['max'],
ndv=0.0)
# add ot a list for subsequent vrt creation
outfiles.append(outfile)
i += 1
for file in outfiles:
return_code = h.check_out_tiff(file)
if return_code != 0:
h.remove_folder_content(temp_dir)
os.remove(file)
return return_code
# write file, so we know this ts has been succesfully processed
if return_code == 0:
with open(str(check_file), 'w') as file:
file.write('passed all tests \n')
# build vrt of timeseries
vrt_options = gdal.BuildVRTOptions(srcNodata=0, separate=True)
gdal.BuildVRT(opj(out_dir, 'Timeseries.{}.{}.vrt'.format(product, pol)),
outfiles,
options=vrt_options)
# remove tmp files
h.delete_dimap(out_stack)
def mt_metrics(stack, out_prefix, metrics, rescale_to_datatype=False,
to_power=False, outlier_removal=False, datelist=None):
# from datetime import datetime
with rasterio.open(stack) as src:
harmonics = False
if 'harmonics' in metrics:
print(' INFO: Calculating harmonics')
if not datelist:
print(' WARNING: Harmonics need the datelist. Harmonics will not be calculated')
else:
harmonics = True
metrics.remove('harmonics')
metrics.extend(['amplitude', 'phase', 'residuals'])
if 'percentiles' in metrics:
metrics.remove('percentiles')
metrics.extend(['p95', 'p5'])
# get metadata
meta = src.profile
# update driver and reduced band count
meta.update({'driver': 'GTiff'})
meta.update({'count': 1})
# write all different output files into a dictionary
metric_dict = {}
for metric in metrics:
filename = '{}.{}.tif'.format(out_prefix, metric)
metric_dict[metric] = rasterio.open(
filename, 'w', **meta)
# scaling factors in case we have to rescale to integer
minimums = {'avg': -30, 'max': -30, 'min': -30,
'std': 0.00001, 'cov': 0.00001}
maximums = {'avg': 5, 'max': 5, 'min': 5, 'std': 15, 'cov': 1}
if harmonics:
# construct independent variables
dates, sines, cosines = [], [], []
two_pi = np.multiply(2, np.pi)
for date in sorted(datelist):
delta = difference_in_years(datetime.strptime('700101',"%y%m%d"), datetime.strptime(date,"%y%m%d"))
dates.append(delta)
sines.append(np.sin(np.multiply(two_pi, delta - 0.5)))
cosines.append(np.cos(np.multiply(two_pi, delta - 0.5)))
X = np.array([dates, cosines, sines])
# loop through blocks
for _, window in src.block_windows(1):
# read array with all bands
stack = src.read(range(1, src.count + 1), window=window)
if rescale_to_datatype is True and meta['dtype'] != 'float32':
stack = ras.rescale_to_float(stack, meta['dtype'])
# transform to power
if to_power is True:
stack = ras.convert_to_power(stack)
# outlier removal (only applies if there are more than 5 bands)
if outlier_removal is True and src.count >= 5:
stack = remove_outliers(stack)
# get stats
arr = {}
arr['p95'], arr['p5'] = (np.nan_to_num(nan_percentile(stack, [95, 5]))
if 'p95' in metrics else (False, False))
arr['median'] = (np.nan_to_num(np.nanmedian(stack, axis=0))
if 'median' in metrics else False)
arr['avg'] = (np.nan_to_num(np.nanmean(stack, axis=0))
if 'avg' in metrics else False)
arr['max'] = (np.nan_to_num(np.nanmax(stack, axis=0))
if 'max' in metrics else False)
arr['min'] = (np.nan_to_num(np.nanmin(stack, axis=0))
if 'min' in metrics else False)
arr['std'] = (np.nan_to_num(np.nanstd(stack, axis=0))
if 'std' in metrics else False)
arr['cov'] = (np.nan_to_num(stats.variation(stack, axis=0,
nan_policy='omit'))
if 'cov' in metrics else False)
if harmonics:
stack_size = (stack.shape[1], stack.shape[2])
if to_power is True:
y = ras.convert_to_db(stack).reshape(stack.shape[0], -1)
else:
y = stack.reshape(stack.shape[0], -1)
x, residuals, _, _ = np.linalg.lstsq(X.T, y)
arr['amplitude'] = np.hypot(x[1], x[2]).reshape(stack_size)
arr['phase'] = np.arctan2(x[2], x[1]).reshape(stack_size)
arr['residuals'] = np.sqrt(np.divide(residuals, stack.shape[0])).reshape(stack_size)
# the metrics to be re-turned to dB, in case to_power is True
metrics_to_convert = ['avg', 'min', 'max', 'p95', 'p5', 'median']
# do the back conversions and write to disk loop
for metric in metrics:
if to_power is True and metric in metrics_to_convert:
arr[metric] = ras.convert_to_db(arr[metric])
if rescale_to_datatype is True and meta['dtype'] != 'float32':
arr[metric] = ras.scale_to_int(arr[metric], meta['dtype'],
minimums[metric],
maximums[metric])
# write to dest
metric_dict[metric].write(
np.float32(arr[metric]), window=window, indexes=1)
metric_dict[metric].update_tags(1,
BAND_NAME='{}_{}'.format(os.path.basename(out_prefix), metric))
metric_dict[metric].set_band_description(1,
'{}_{}'.format(os.path.basename(out_prefix), metric))
# close the output files
for metric in metrics:
# close rio opening
metric_dict[metric].close()
# construct filename
filename = '{}.{}.tif'.format(out_prefix, metric)
return_code = h.check_out_tiff(filename)
if return_code != 0:
# remove all files and return
for metric in metrics:
filename = '{}.{}.tif'.format(out_prefix, metric)
os.remove(filename)
return return_code
if return_code == 0:
dirname = os.path.dirname(out_prefix)
check_file = opj(dirname, '.{}.processed'.format(os.path.basename(out_prefix)))
with open(str(check_file), 'w') as file:
file.write('passed all tests \n')
def mosaic(filelist, outfile, config_file, cut_to_aoi=None, harm=None):
if outfile.parent.joinpath(f'.{outfile.name[:-4]}.processed').exists():
logger.info(f'{outfile} already exists.')
return
logger.info(f'Mosaicking file {outfile}.')
with open(config_file, 'r') as ard_file:
config_dict = json.load(ard_file)
temp_dir = config_dict['temp_dir']
aoi = config_dict['aoi']
epsg = config_dict['processing']['single_ARD']['dem']['out_projection']
if not harm:
harm = config_dict['processing']['mosaic']['harmonization']
if not cut_to_aoi:
cut_to_aoi = config_dict['processing']['mosaic']['cut_to_aoi']
logfile = outfile.parent.joinpath(f'{str(outfile)[:-4]}.errLog')
with TemporaryDirectory(prefix=f'{temp_dir}/') as temp:
temp = Path(temp)
# get datatype from first image in our mosaic filelist
with rasterio.open(filelist.split(' ')[0]) as src:
dtype = src.meta['dtype']
dtype = 'float' if dtype == 'float32' else dtype
if cut_to_aoi:
tempfile = temp.joinpath(outfile.name)
else:
tempfile = outfile
harm = 'band' if harm else 'none'
cmd = (
f"otbcli_Mosaic -ram 8192 -progress 1 "
f"-comp.feather large "
f"-harmo.method {harm} "
f"-harmo.cost rmse "
f"-tmpdir {str(temp)} "
f"-interpolator bco"
f" -il {filelist} "
f" -out {str(tempfile)} {dtype}"
)
return_code = h.run_command(cmd, logfile)
if return_code != 0:
if tempfile.exists():
tempfile.unlink()
return
if cut_to_aoi:
# get aoi in a way rasterio wants it
aoi_gdf = vec.wkt_to_gdf(aoi)
features = vec.gdf_to_json_geometry(aoi_gdf.to_crs(epsg=epsg))
# import raster and mask
with rasterio.open(tempfile) as src:
out_image, out_transform = rasterio.mask.mask(src, features,
crop=True)
out_meta = src.meta.copy()
ndv = src.nodata
out_image = np.ma.masked_where(out_image == ndv, out_image)
out_meta.update({
'driver': 'GTiff',
'height': out_image.shape[1],
'width': out_image.shape[2],
'transform': out_transform,
'tiled': True,
'blockxsize': 128,
'blockysize': 128
})
with rasterio.open(outfile, 'w', **out_meta) as dest:
dest.write(out_image.data)
# remove intermediate file
tempfile.unlink()
# check
return_code = h.check_out_tiff(outfile)
if return_code != 0:
if outfile.exists():
outfile.unlink()
else:
check_file = outfile.parent.joinpath(
f'.{outfile.name[:-4]}.processed'
)
with open(str(check_file), 'w') as file:
file.write('passed all tests \n')