def create_ard(self, infile, out_dir, subset=None, overwrite=False):
"""
:param infile:
:param out_dir:
:param subset:
:param overwrite:
:return:
"""
if self.product_type != 'GRD':
raise ValueError(
'The create_ard method for single products is currently '
'only available for GRD products')
if isinstance(infile, str):
infile = Path(infile)
if isinstance(out_dir, str):
out_dir = Path(out_dir)
# set config param necessary for processing
self.config_dict['processing_dir'] = str(out_dir)
self.config_dict['temp_dir'] = str(out_dir.joinpath('temp'))
self.config_dict['snap_cpu_parallelism'] = os.cpu_count()
self.config_dict['subset'] = False
if subset:
self.config_dict['subset'] = True
self.config_dict['aoi'] = subset
# create directories
out_dir.mkdir(parents=True, exist_ok=True)
out_dir.joinpath('temp').mkdir(parents=True, exist_ok=True)
if overwrite:
file_dir = out_dir.joinpath(f'{self.rel_orbit}/{self.start_date}')
if file_dir.joinpath('.processed').exists():
file_dir.joinpath('.processed').unlink()
# --------------------------------------------
# 2 Check if within SRTM coverage
# set ellipsoid correction and force GTC production
# when outside SRTM
center_lat = self._get_center_lat(infile)
if float(center_lat) > 59 or float(center_lat) < -59:
logger.info(
'Scene is outside SRTM coverage. Snap will therefore use '
'the GETASSE30 DEM. Also consider to use a stereographic '
'projection.')
epsg = input(
'Please type the EPSG you want to project the output data or '
'just press enter for keeping Lat/Lon coordinate system '
'(e.g. 3413 for NSIDC Sea Ice Polar Stereographic North '
'projection, or 3976 for NSIDC Sea Ice Polar Stereographic '
'South projection')
if not epsg:
epsg = 4326
self.ard_parameters['single_ARD']['dem']['dem_name'] = 'GETASSE30'
self.ard_parameters['single_ARD']['dem']['out_projection'] = int(
epsg)
# --------------------------------------------
# 3 Check ard parameters in case they have been updated,
# and write them to json file
# set config file to output directory
self.config_file = out_dir.joinpath('processing.json')
# write ard parameters, and check if they are correct
self.update_ard_parameters()
# --------------------------------------------
# 4 set resolution to degree
# self.ard_parameters['resolution'] = h.resolution_in_degree(
# self.center_lat, self.ard_parameters['resolution'])
# --------------------------------------------
# 5 run the burst to ard batch routine
filelist, out_bs, out_ls, error = grd_to_ard(
[infile],
self.config_file,
)
# print error if any
if error:
logger.info(error)
else:
# remove temp folder
h.remove_folder_content(out_dir.joinpath('temp'))
out_dir.joinpath('temp').rmdir()
self.ard_dimap = out_bs
def grds_to_ard(self,
inventory_df=None,
subset=None,
timeseries=False,
timescan=False,
mosaic=False,
overwrite=False,
exec_file=None,
cut_to_aoi=False):
self.update_ard_parameters()
if overwrite:
print(' INFO: Deleting processing folder to start from scratch')
h.remove_folder_content(self.processing_dir)
# set resolution in degree
# self.center_lat = loads(self.aoi).centroid.y
# if float(self.center_lat) > 59 or float(self.center_lat) < -59:
# print(' INFO: Scene is outside SRTM coverage. Will use 30m ASTER'
# ' DEM instead.')
# self.ard_parameters['dem'] = 'ASTER 1sec GDEM'
if subset:
if subset.split('.')[-1] == '.shp':
subset = str(vec.shp_to_wkt(subset, buffer=0.1, envelope=True))
elif subset.startswith('POLYGON (('):
subset = loads(subset).buffer(0.1).to_wkt()
else:
print(
' ERROR: No valid subset given.'
' Should be either path to a shapefile or a WKT Polygon.')
sys.exit()
# check number of already prcessed acquisitions
nr_of_processed = len(
glob.glob(opj(self.processing_dir, '*', '20*', '.processed')))
# number of acquisitions to process
nr_of_acq = len(
inventory_df.groupby(['relativeorbit', 'acquisitiondate']))
# check and retry function
i = 0
while nr_of_acq > nr_of_processed:
# the grd to ard batch routine
grd_batch.grd_to_ard_batch(inventory_df, self.download_dir,
self.processing_dir, self.temp_dir,
self.proc_file, subset, self.data_mount,
exec_file)
# reset number of already processed acquisitions
nr_of_processed = len(
glob.glob(opj(self.processing_dir, '*', '20*', '.processed')))
i += 1
# not more than 5 trys
if i == 5:
break
# time-series part
if timeseries or timescan:
nr_of_processed = len(
glob.glob(
opj(self.processing_dir, '*', 'Timeseries',
'.*processed')))
nr_of_polar = len(
inventory_df.polarisationmode.unique()[0].split(' '))
nr_of_tracks = len(inventory_df.relativeorbit.unique())
nr_of_ts = nr_of_polar * nr_of_tracks
# check and retry function
i = 0
while nr_of_ts > nr_of_processed:
grd_batch.ards_to_timeseries(inventory_df, self.processing_dir,
self.temp_dir, self.proc_file,
exec_file)
nr_of_processed = len(
glob.glob(
opj(self.processing_dir, '*', 'Timeseries',
'.*processed')))
i += 1
# not more than 5 trys
if i == 5:
break
if timescan:
# number of already processed timescans
nr_of_processed = len(
glob.glob(
opj(self.processing_dir, '*', 'Timescan', '.*processed')))
# number of expected timescans
nr_of_polar = len(
inventory_df.polarisationmode.unique()[0].split(' '))
nr_of_tracks = len(inventory_df.relativeorbit.unique())
nr_of_ts = nr_of_polar * nr_of_tracks
i = 0
while nr_of_ts > nr_of_processed:
grd_batch.timeseries_to_timescan(inventory_df,
self.processing_dir,
self.proc_file)
nr_of_processed = len(
glob.glob(
opj(self.processing_dir, '*', 'Timescan',
'.*processed')))
i += 1
# not more than 5 trys
if i == 5:
break
if i < 5 and exec_file:
print(' create vrt command')
if cut_to_aoi:
cut_to_aoi = self.aoi
if mosaic and timeseries and not subset:
grd_batch.mosaic_timeseries(inventory_df, self.processing_dir,
self.temp_dir, cut_to_aoi)
if mosaic and timescan and not subset:
grd_batch.mosaic_timescan(inventory_df, self.processing_dir,
self.temp_dir, self.proc_file,
cut_to_aoi)
def burst_to_ard(master_file,
swath,
master_burst_nr,
master_burst_id,
proc_file,
out_dir,
temp_dir,
slave_file=None,
slave_burst_nr=None,
slave_burst_id=None,
coherence=False,
remove_slave_import=False):
'''The main routine to turn a burst into an ARD product
Args:
master_file (str): path to full master SLC scene
swath (str): subswath
master_burst_nr (): index number of the burst
master_burst_id ():
out_dir (str):
temp_dir (str):
slave_file (str):
slave_burst_nr (str):
slave_burst_id (str):
proc_file (str):
remove_slave_import (bool):
'''
# load ards
with open(proc_file, 'r') as ard_file:
ard_params = json.load(ard_file)['processing parameters']
ard = ard_params['single ARD']
# import master
master_import = opj(temp_dir, '{}_import'.format(master_burst_id))
if not os.path.exists('{}.dim'.format(master_import)):
import_log = opj(out_dir, '{}_import.err_log'.format(master_burst_id))
polars = ard['polarisation'].replace(' ', '')
return_code = _import(master_file, master_import, import_log, swath,
master_burst_nr, polars)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
if ard['H-A-Alpha']:
# create HAalpha file
out_haa = opj(temp_dir, '{}_h'.format(master_burst_id))
haa_log = opj(out_dir, '{}_haa.err_log'.format(master_burst_id))
return_code = _ha_alpha('{}.dim'.format(master_import), out_haa,
haa_log, ard['remove pol speckle'])
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# geo code HAalpha
out_htc = opj(temp_dir, '{}_pol'.format(master_burst_id))
haa_tc_log = opj(out_dir, '{}_haa_tc.err_log'.format(master_burst_id))
return_code = _terrain_correction('{}.dim'.format(out_haa), out_htc,
haa_tc_log, ard['resolution'],
ard['dem'])
# last check on the output files
return_code = h.check_out_dimap(out_htc)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# move to final destination
h.move_dimap(out_htc, opj(out_dir, '{}_pol'.format(master_burst_id)))
# remove HAalpha tmp files
h.delete_dimap(out_haa)
# calibrate
out_cal = opj(temp_dir, '{}_cal'.format(master_burst_id))
cal_log = opj(out_dir, '{}_cal.err_log'.format(master_burst_id))
return_code = _calibration('{}.dim'.format(master_import), out_cal,
cal_log, ard['product type'])
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
if not coherence:
# remove imports
h.delete_dimap(master_import)
# speckle filtering
if ard['remove speckle']:
speckle_import = opj(temp_dir,
'{}_speckle_import'.format(master_burst_id))
speckle_log = opj(out_dir,
'{}_speckle.err_log'.format(master_burst_id))
return_code = _speckle_filter('{}.dim'.format(out_cal), speckle_import,
speckle_log)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# remove temp file
h.delete_dimap(out_cal)
# reset master_import for follwoing routine
out_cal = speckle_import
# do terrain flattening in case it is selected
if ard['product type'] == 'RTC':
# define outfile
out_rtc = opj(temp_dir, '{}_rtc'.format(master_burst_id))
rtc_log = opj(out_dir, '{}_rtc.err_log'.format(master_burst_id))
# do the TF
return_code = _terrain_flattening('{}.dim'.format(out_cal), out_rtc,
rtc_log, ard['dem'])
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# remove tmp files
h.delete_dimap(out_cal)
# set out_rtc to out_cal for further processing
out_cal = out_rtc
if ard['to db']:
out_db = opj(temp_dir, '{}_cal_db'.format(master_burst_id))
db_log = opj(out_dir, '{}_cal_db.err_log'.format(master_burst_id))
return_code = _linear_to_db('{}.dim'.format(out_cal), out_db, db_log)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# remove tmp files
h.delete_dimap(out_cal)
# set out_cal to out_db for further processing
out_cal = out_db
# geo code backscatter products
out_tc = opj(temp_dir, '{}_bs'.format(master_burst_id))
tc_log = opj(out_dir, '{}_bs_tc.err_log'.format(master_burst_id))
return_code = _terrain_correction('{}.dim'.format(out_cal), out_tc, tc_log,
ard['resolution'], ard['dem'])
# last check on backscatter data
return_code = h.check_out_dimap(out_tc)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# we move backscatter to final destination
h.move_dimap(out_tc, opj(out_dir, '{}_bs'.format(master_burst_id)))
if ard['create ls mask']:
# create LS map
out_ls = opj(temp_dir, '{}_LS'.format(master_burst_id))
ls_log = opj(out_dir, '{}_LS.err_log'.format(master_burst_id))
return_code = _ls_mask('{}.dim'.format(out_cal), out_ls, ls_log,
ard['resolution'], ard['dem'])
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# last check on ls data
return_code = h.check_out_dimap(out_ls, test_stats=False)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# move ls data to final destination
h.move_dimap(out_ls, opj(out_dir, '{}_LS'.format(master_burst_id)))
# remove calibrated files
h.delete_dimap(out_cal)
if coherence:
# import slave
slave_import = opj(temp_dir, '{}_import'.format(slave_burst_id))
import_log = opj(out_dir, '{}_import.err_log'.format(slave_burst_id))
polars = ard['polarisation'].replace(' ', '')
return_code = _import(slave_file, slave_import, import_log, swath,
slave_burst_nr, polars)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# co-registration
#filelist = ['{}.dim'.format(master_import),
# '{}.dim'.format(slave_import)]
#filelist = '\'{}\''.format(','.join(filelist))
out_coreg = opj(temp_dir, '{}_coreg'.format(master_burst_id))
coreg_log = opj(out_dir, '{}_coreg.err_log'.format(master_burst_id))
# return_code = _coreg(filelist, out_coreg, coreg_log, dem)
return_code = _coreg2('{}.dim'.format(master_import),
'{}.dim'.format(slave_import), out_coreg,
coreg_log, ard['dem'])
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# remove imports
h.delete_dimap(master_import)
if remove_slave_import is True:
h.delete_dimap(slave_import)
# calculate coherence and deburst
out_coh = opj(temp_dir, '{}_c'.format(master_burst_id))
coh_log = opj(out_dir, '{}_coh.err_log'.format(master_burst_id))
coh_polars = ard['coherence bands'].replace(' ', '')
return_code = _coherence('{}.dim'.format(out_coreg), out_coh, coh_log,
coh_polars)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# remove coreg tmp files
h.delete_dimap(out_coreg)
# geocode
out_tc = opj(temp_dir, '{}_coh'.format(master_burst_id))
tc_log = opj(out_dir, '{}_coh_tc.err_log'.format(master_burst_id))
return_code = _terrain_correction('{}.dim'.format(out_coh), out_tc,
tc_log, ard['resolution'],
ard['dem'])
# last check on coherence data
return_code = h.check_out_dimap(out_tc)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# move to final destination
h.move_dimap(out_tc, opj(out_dir, '{}_coh'.format(master_burst_id)))
# remove tmp files
h.delete_dimap(out_coh)
# write file, so we know this burst has been succesfully processed
if return_code == 0:
check_file = opj(out_dir, '.processed')
with open(str(check_file), 'w') as file:
file.write('passed all tests \n')
else:
h.remove_folder_content(temp_dir)
h.remove_folder_content(out_dir)
return return_code
def grd_to_ard(filelist,
output_dir,
file_id,
temp_dir,
proc_file,
subset=None):
'''The main function for the grd to ard generation
This function represents the full workflow for the generation of an
Analysis-Ready-Data product. The standard parameters reflect the CEOS
ARD defintion for Sentinel-1 backcsatter products.
By changing the parameters, taking care of all parameters
that can be given. The function can handle multiple inputs of the same
acquisition, given that there are consecutive data takes.
Args:
filelist (list): must be a list with one or more absolute
paths to GRD scene(s)
output_dir: os.path object or string for the folder
where the output file should be written#
file_id (str): prefix of the final output file
temp_dir:
resolution: the resolution of the output product in meters
ls_mask: layover/shadow mask generation (Boolean)
speckle_filter: speckle filtering (Boolean)
Returns:
nothing
Notes:
no explicit return value, since output file is our actual return
'''
# load ard parameters
with open(proc_file, 'r') as ard_file:
ard_params = json.load(ard_file)['processing parameters']
ard = ard_params['single ARD']
polars = ard['polarisation'].replace(' ', '')
# slice assembly if more than one scene
if len(filelist) > 1:
for file in filelist:
grd_import = opj(temp_dir,
'{}_imported'.format(os.path.basename(file)[:-5]))
logfile = opj(
output_dir,
'{}.Import.errLog'.format(os.path.basename(file)[:-5]))
return_code = _grd_frame_import(file, grd_import, logfile, polars)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# create list of scenes for full acquisition in
# preparation of slice assembly
scenelist = ' '.join(glob.glob(opj(temp_dir, '*imported.dim')))
# create file strings
grd_import = opj(temp_dir, '{}_imported'.format(file_id))
logfile = opj(output_dir, '{}._slice_assembly.errLog'.format(file_id))
return_code = _slice_assembly(
scenelist,
grd_import,
logfile,
)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
for file in filelist:
h.delete_dimap(
opj(temp_dir,
'{}_imported'.format(os.path.basename(str(file))[:-5])))
if subset:
grd_subset = opj(temp_dir, '{}_imported_subset'.format(file_id))
return_code = _grd_subset_georegion('{}.dim'.format(grd_import),
grd_subset, logfile, subset)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# delete slice assembly
h.delete_dimap(grd_import)
# single scene case
else:
grd_import = opj(temp_dir, '{}_imported'.format(file_id))
logfile = opj(output_dir, '{}.Import.errLog'.format(file_id))
if subset is None:
return_code = _grd_frame_import(filelist[0], grd_import, logfile,
polars)
else:
return_code = _grd_frame_import_subset(filelist[0], grd_import,
subset, logfile, polars)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# ---------------------------------------------------------------------
# Remove the grd border noise from existent channels (OST routine)
if ard['remove border noise'] and not subset:
for polarisation in ['VV', 'VH', 'HH', 'HV']:
infile = glob.glob(
opj(temp_dir, '{}_imported*data'.format(file_id),
'Intensity_{}.img'.format(polarisation)))
if len(infile) == 1:
# run grd Border Remove
print(' INFO: Remove border noise for {} band.'.format(
polarisation))
_grd_remove_border(infile[0])
# set new infile
infile = glob.glob(opj(temp_dir, '{}_imported*dim'.format(file_id)))[0]
# -------------------------------------------
# in case we want to apply Speckle filtering
if ard['remove speckle']:
logfile = opj(temp_dir, '{}.Speckle.errLog'.format(file_id))
outfile = opj(temp_dir, '{}_spk'.format(file_id))
# run processing
return_code = _grd_speckle_filter(infile, outfile, logfile)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# delete input
h.delete_dimap(infile[:-4])
# define infile for next processing step
infile = '{}.dim'.format(outfile)
# ----------------------
# do the calibration
outfile = opj(temp_dir, '{}.{}'.format(file_id, ard['product type']))
logfile = opj(output_dir, '{}.Backscatter.errLog'.format(file_id))
return_code = _grd_backscatter(infile, outfile, logfile, ard['dem'],
ard['product type'])
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# delete input file
h.delete_dimap(infile[:-4])
# input file for follwoing
infile = '{}.dim'.format(outfile)
# ----------------------------------------------
# let's create a Layover shadow mask if needed
if ard['create ls mask'] is True:
outfile = opj(temp_dir, '{}.ls_mask'.format(file_id))
logfile = opj(output_dir, '{}.ls_mask.errLog'.format(file_id))
return_code = _grd_ls_mask(infile, outfile, logfile, ard['resolution'],
ard['dem'])
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# last check on ls data
return_code = h.check_out_dimap(outfile, test_stats=False)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# move to final destination
out_ls_mask = opj(output_dir, '{}.LS'.format(file_id))
# delete original file sin case they exist
if os.path.exists(str(out_ls_mask) + '.dim'):
h.delete_dimap(out_ls_mask)
# move out of temp
shutil.move('{}.dim'.format(outfile), '{}.dim'.format(out_ls_mask))
shutil.move('{}.data'.format(outfile), '{}.data'.format(out_ls_mask))
# to db
if ard['to db']:
logfile = opj(output_dir, '{}.linToDb.errLog'.format(file_id))
outfile = opj(temp_dir, '{}_{}_db'.format(file_id,
ard['product type']))
return_code = _grd_to_db(infile, outfile, logfile)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# delete
h.delete_dimap(infile[:-4])
# re-define infile
infile = opj(temp_dir, '{}_{}_db.dim'.format(file_id,
ard['product type']))
# -----------------------
# let's geocode the data
# infile = opj(temp_dir, '{}.{}.dim'.format(file_id, product_type))
outfile = opj(temp_dir, '{}.bs'.format(file_id))
logfile = opj(output_dir, '{}.bs.errLog'.format(file_id))
return_code = _grd_terrain_correction(infile, outfile, logfile,
ard['resolution'], ard['dem'])
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# remove calibrated files
h.delete_dimap(infile[:-4])
# move to final destination
out_final = opj(output_dir, '{}.bs'.format(file_id))
# remove file if exists
if os.path.exists(out_final + '.dim'):
h.delete_dimap(out_final)
return_code = h.check_out_dimap(outfile)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
shutil.move('{}.dim'.format(outfile), '{}.dim'.format(out_final))
shutil.move('{}.data'.format(outfile), '{}.data'.format(out_final))
# write file, so we know this burst has been succesfully processed
if return_code == 0:
check_file = opj(output_dir, '.processed')
with open(str(check_file), 'w') as file:
file.write('passed all tests \n')
else:
h.remove_folder_content(temp_dir)
h.remove_folder_content(output_dir)
def bursts_to_ard(self,
timeseries=False,
timescan=False,
mosaic=False,
overwrite=False,
exec_file=None,
cut_to_aoi=False):
# in case ard parameters have been updated, write them to json file
self.update_ard_parameters()
if overwrite:
print(' INFO: Deleting processing folder to start from scratch')
h.remove_folder_content(self.processing_dir)
# set resolution in degree
self.center_lat = loads(self.aoi).centroid.y
if float(self.center_lat) > 59 or float(self.center_lat) < -59:
print(' INFO: Scene is outside SRTM coverage. Will use 30m ASTER'
' DEM instead.')
self.ard_parameters['single ARD']['dem'] = 'ASTER 1sec GDEM'
# set resolution to degree
# self.ard_parameters['resolution'] = h.resolution_in_degree(
# self.center_lat, self.ard_parameters['resolution'])
nr_of_processed = len(
glob.glob(opj(self.processing_dir, '*', '*', '.processed')))
# check and retry function
i = 0
while len(self.burst_inventory) > nr_of_processed:
burst.burst_to_ard_batch(self.burst_inventory, self.download_dir,
self.processing_dir, self.temp_dir,
self.proc_file, self.data_mount,
exec_file)
nr_of_processed = len(
glob.glob(opj(self.processing_dir, '*', '*', '.processed')))
i += 1
# not more than 5 trys
if i == 5:
break
# do we delete the downloads here?
if timeseries or timescan:
burst.burst_ards_to_timeseries(self.burst_inventory,
self.processing_dir, self.temp_dir,
self.proc_file, exec_file)
# do we deleete the single ARDs here?
if timescan:
burst.timeseries_to_timescan(self.burst_inventory,
self.processing_dir,
self.temp_dir, self.proc_file)
if cut_to_aoi:
cut_to_aoi = self.aoi
if mosaic and timeseries:
burst.mosaic_timeseries(self.burst_inventory, self.processing_dir,
self.temp_dir, cut_to_aoi)
if mosaic and timescan:
burst.mosaic_timescan(self.burst_inventory, self.processing_dir,
self.temp_dir, self.proc_file, cut_to_aoi)
def grd_to_ard(filelist, config_file):
"""Main function for the grd to ard generation
This function represents the full workflow for the generation of an
Analysis-Ready-Data product. The standard parameters reflect the CEOS
ARD defintion for Sentinel-1 backcsatter products.
By changing the parameters, taking care of all parameters
that can be given. The function can handle multiple inputs of the same
acquisition, given that there are consecutive data takes.
:param filelist: must be a list with one or more
absolute paths to GRD scene(s)
:param config_file:
:return:
"""
from ost.s1.s1scene import Sentinel1Scene
# ----------------------------------------------------
# 1 load relevant config parameters
with open(config_file, 'r') as file:
config_dict = json.load(file)
ard = config_dict['processing']['single_ARD']
processing_dir = Path(config_dict['processing_dir'])
subset = config_dict['subset']
# ----------------------------------------------------
# 2 define final destination dir/file and ls mask
# get acq data and track from first scene in list
first = Sentinel1Scene(Path(filelist[0]).stem)
acquisition_date = first.start_date
track = first.rel_orbit
logger.info(
f'Processing acquisition from {acquisition_date} over track {track}.'
)
# construct namespace for out directory etc.
out_dir = processing_dir.joinpath(f'{track}/{acquisition_date}')
out_dir.mkdir(parents=True, exist_ok=True)
file_id = f'{acquisition_date}_{track}'
out_final = out_dir.joinpath(f'{file_id}_bs')
out_ls_mask = out_dir.joinpath(f'{file_id}_LS')
suf = '.tif' if ard['to_tif'] else '.dim'
# ----------------------------------------------------
# 3 check if already processed
if out_dir.joinpath('.processed').exists() and \
out_final.with_suffix(suf).exists():
logger.info(
f'Acquisition from {acquisition_date} of track {track} '
f'already processed'
)
if out_ls_mask.with_suffix(suf).exists():
out_ls = out_ls_mask.with_suffix(suf)
else:
out_ls = None
return filelist, out_final.with_suffix(suf), out_ls, None
# ----------------------------------------------------
# 4 run the processing routine
# this might happen in the create_ard from s1scene class
if not config_dict['temp_dir']:
temp_dir = processing_dir.joinpath('temp')
temp_dir.mkdir(parents=True, exist_ok=True)
else:
temp_dir = config_dict['temp_dir']
with TemporaryDirectory(prefix=f"{temp_dir}/") as temp:
# convert temp directory to Path object
temp = Path(temp)
# ---------------------------------------------------------------------
# 4.1 Import
# slice assembly if more than one scene
if len(filelist) > 1:
# if more than one frame import all files
for file in filelist:
# unzip for faster import?
unpack = None
if Path(file).suffix == '.zip':
with zipfile.ZipFile(file, 'r') as zip_ref:
zip_ref.extractall(temp)
file = temp.joinpath(f'{file.stem}.SAFE')
unpack = True
# create namespace for temporary imported product
grd_import = temp.joinpath(f'{file.stem}_imported')
# create namespace for import log
logfile = out_dir.joinpath(f'{file.stem}.Import.errLog')
# set subset tempraoally to false for import routine
config_dict['subset'] = False
# frame import
try:
grd.grd_frame_import(
file, grd_import, logfile, config_dict
)
except (GPTRuntimeError, NotValidFileError) as error:
logger.info(error)
return filelist, None, None, error
config_dict['subset'] = subset
if unpack:
h.remove_folder_content(file)
file.rmdir()
# create list of scenes for full acquisition in
# preparation of slice assembly
scenelist = ' '.join(
[str(file) for file in list(temp.glob('*imported.dim'))]
)
# create namespace for temporary slice assembled import product
grd_import = temp.joinpath(f'{file_id}_imported')
# create namespace for slice assembled log
logfile = out_dir.joinpath(f'{file_id}._slice_assembly.errLog')
# run slice assembly
try:
grd.slice_assembly(scenelist, grd_import, logfile, config_dict)
except (GPTRuntimeError, NotValidFileError) as error:
logger.info(error)
return filelist, None, None, error
# delete imported frames
for file in filelist:
h.delete_dimap(temp.joinpath(f'{file.stem}_imported'))
# subset mode after slice assembly
if subset:
# create namespace for temporary subset product
grd_subset = temp.joinpath(f'{file_id}_imported_subset')
# create namespace for subset log
logfile = out_dir.joinpath(f'{file_id}._slice_assembly.errLog')
# run subset routine
try:
grd.grd_subset_georegion(
grd_import.with_suffix('.dim'), grd_subset, logfile,
config_dict
)
except (GPTRuntimeError, NotValidFileError) as error:
logger.info(error)
return filelist, None, None, error
# delete slice assembly input to subset
h.delete_dimap(grd_import)
# set subset to import for subsequent functions
grd_import = grd_subset
# single scene case
else:
file = filelist[0]
# unzip for faster import
unpack = None
if Path(file).suffix == '.zip':
with zipfile.ZipFile(file, 'r') as zip_ref:
zip_ref.extractall(temp)
file = temp.joinpath(f'{file.stem}.SAFE')
unpack = True
# create namespace for temporary imported product
grd_import = temp.joinpath(f'{file_id}_imported')
# create namespace for import log
logfile = out_dir.joinpath(f'{file_id}.Import.errLog')
# run frame import
try:
grd.grd_frame_import(file, grd_import, logfile, config_dict)
except (GPTRuntimeError, NotValidFileError) as error:
logger.info(error)
return filelist, None, None, error
if unpack:
h.remove_folder_content(file)
file.rmdir()
# set input for next step
infile = grd_import.with_suffix('.dim')
# ---------------------------------------------------------------------
# 4.2 GRD Border Noise
if ard['remove_border_noise'] and not subset:
# loop through possible polarisations
for polarisation in ['VV', 'VH', 'HH', 'HV']:
# get input file
file = list(temp.glob(
f'{file_id}_imported*data/Intensity_{polarisation}.img'
))
# remove border noise
if len(file) == 1:
# run grd Border Remove
grd.grd_remove_border(file[0])
# ---------------------------------------------------------------------
# 4.3 Calibration
# create namespace for temporary calibrated product
calibrated = temp.joinpath(f'{file_id}_cal')
# create namespace for calibration log
logfile = out_dir.joinpath(f'{file_id}.calibration.errLog')
# run calibration
try:
grd.calibration(infile, calibrated, logfile, config_dict)
except (GPTRuntimeError, NotValidFileError) as error:
logger.info(error)
return filelist, None, None, error
# delete input
h.delete_dimap(infile.with_suffix(''))
# input for next step
infile = calibrated.with_suffix('.dim')
# ---------------------------------------------------------------------
# 4.4 Multi-looking
if int(ard['resolution']) >= 20:
# create namespace for temporary multi-looked product
multi_looked = temp.joinpath(f'{file_id}_ml')
# create namespace for multi-loook log
logfile = out_dir.joinpath(f'{file_id}.multilook.errLog')
# run multi-looking
try:
grd.multi_look(infile, multi_looked, logfile, config_dict)
except (GPTRuntimeError, NotValidFileError) as error:
logger.info(error)
return filelist, None, None, error
# delete input
h.delete_dimap(infile.with_suffix(''))
# define input for next step
infile = multi_looked.with_suffix('.dim')
# ---------------------------------------------------------------------
# 4.5 Layover shadow mask
out_ls = None # set to none for final return statement
if ard['create_ls_mask'] is True:
# create namespace for temporary ls mask product
ls_mask = temp.joinpath(f'{file_id}_ls_mask')
# create namespace for ls mask log
logfile = out_dir.joinpath(f'{file_id}.ls_mask.errLog')
# run ls mask routine
try:
common.ls_mask(infile, ls_mask, logfile, config_dict)
out_ls = out_ls_mask.with_suffix('.dim')
except (GPTRuntimeError, NotValidFileError) as error:
logger.info(error)
return filelist, None, None, error
# polygonize
ls_raster = list(ls_mask.with_suffix('.data').glob('*img'))[0]
ras.polygonize_ls(ls_raster, ls_mask.with_suffix('.json'))
# ---------------------------------------------------------------------
# 4.6 Speckle filtering
if ard['remove_speckle']:
# create namespace for temporary speckle filtered product
filtered = temp.joinpath(f'{file_id}_spk')
# create namespace for speckle filter log
logfile = out_dir.joinpath(f'{file_id}.Speckle.errLog')
# run speckle filter
try:
common.speckle_filter(infile, filtered, logfile, config_dict)
except (GPTRuntimeError, NotValidFileError) as error:
logger.info(error)
return filelist, None, None, error
# delete input
h.delete_dimap(infile.with_suffix(''))
# define input for next step
infile = filtered.with_suffix('.dim')
# ---------------------------------------------------------------------
# 4.7 Terrain flattening
if ard['product_type'] == 'RTC-gamma0':
# create namespace for temporary terrain flattened product
flattened = temp.joinpath(f'{file_id}_flat')
# create namespace for terrain flattening log
logfile = out_dir.joinpath(f'{file_id}.tf.errLog')
# run terrain flattening
try:
common.terrain_flattening(
infile, flattened, logfile, config_dict
)
except (GPTRuntimeError, NotValidFileError) as error:
logger.info(error)
return filelist, None, None, error
# delete input file
h.delete_dimap(infile.with_suffix(''))
# define input for next step
infile = flattened.with_suffix('.dim')
# ---------------------------------------------------------------------
# 4.8 Linear to db
if ard['to_db']:
# create namespace for temporary db scaled product
db_scaled = temp.joinpath(f'{file_id}_db')
# create namespace for db scaled log
logfile = out_dir.joinpath(f'{file_id}.db.errLog')
# run db scaling routine
try:
common.linear_to_db(infile, db_scaled, logfile, config_dict)
except (GPTRuntimeError, NotValidFileError) as error:
logger.info(error)
return filelist, None, None, error
# delete input file
h.delete_dimap(infile.with_suffix(''))
# set input for next step
infile = db_scaled.with_suffix('.dim')
# ---------------------------------------------------------------------
# 4.9 Geocoding
# create namespace for temporary geocoded product
geocoded = temp.joinpath(f'{file_id}_bs')
# create namespace for geocoding log
logfile = out_dir.joinpath(f'{file_id}_bs.errLog')
# run geocoding
try:
common.terrain_correction(infile, geocoded, logfile, config_dict)
except (GPTRuntimeError, NotValidFileError) as error:
logger.info(error)
return filelist, None, None, error
# delete input file
h.delete_dimap(infile.with_suffix(''))
# define final destination
out_final = out_dir.joinpath(f'{file_id}_bs')
# ---------------------------------------------------------------------
# 4.11 Create an outline
ras.image_bounds(geocoded.with_suffix('.data'))
# ---------------------------------------------------------------------
# 4.11 Copy LS Mask vector to data dir
if ard['create_ls_mask'] is True:
ls_mask.with_suffix('.json').rename(
geocoded.with_suffix('.data')
.joinpath(ls_mask.name).with_suffix('.json')
)
# ---------------------------------------------------------------------
# 4.12 Move to output directory
h.move_dimap(geocoded, out_final, ard['to_tif'])
# ---------------------------------------------------------------------
# 5 write processed file to keep track of files already processed
with open(out_dir.joinpath('.processed'), 'w') as file:
file.write('passed all tests \n')
return filelist, out_final.with_suffix('.dim'), out_ls, None
def burst_to_ard(master_file,
swath,
master_burst_nr,
master_burst_id,
out_dir,
temp_dir,
slave_file=None,
slave_burst_nr=None,
slave_burst_id=None,
coherence=False,
polarimetry=False,
pol_speckle_filter=False,
resolution=20,
product_type='GTCgamma',
speckle_filter=False,
to_db=False,
ls_mask_create=False,
dem='SRTM 1sec HGT',
remove_slave_import=False):
'''The main routine to turn a burst into an ARD product
Args:
master_file (str): path to full master SLC scene
swath (str): subswath
master_burst_nr (): index number of the burst
master_burst_id ():
out_dir (str):
temp_dir (str):
slave_file (str):
slave_burst_nr (str):
slave_burst_id (str):
coherence (bool):
polarimetry (bool):
pol_speckle_filter (bool):
resolution (int):
product_type (str):
speckle_filter (bool):
to_db (bool):
ls_mask (bool):
dem (str):
remove_slave_import (bool):
'''
# import master
master_import = opj(temp_dir, '{}_import'.format(master_burst_id))
if not os.path.exists('{}.dim'.format(master_import)):
import_log = opj(out_dir, '{}_import.err_log'.format(master_burst_id))
return_code = _import(master_file, master_import, import_log, swath,
master_burst_nr)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
if polarimetry:
# create HAalpha file
out_haa = opj(temp_dir, '{}_h'.format(master_burst_id))
haa_log = opj(out_dir, '{}_haa.err_log'.format(master_burst_id))
return_code = _ha_alpha('{}.dim'.format(master_import), out_haa,
haa_log, pol_speckle_filter)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# geo code HAalpha
out_htc = opj(temp_dir, '{}_ha_alpha'.format(master_burst_id))
haa_tc_log = opj(out_dir, '{}_haa_tc.err_log'.format(master_burst_id))
return_code = _terrain_correction('{}.dim'.format(out_haa), out_htc,
haa_tc_log, resolution, dem)
# last check on the output files
return_code = h.check_out_dimap(out_htc)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# move to final destination
h.move_dimap(out_htc,
opj(out_dir, '{}_ha_alpha'.format(master_burst_id)))
# remove HAalpha tmp files
h.delete_dimap(out_haa)
# calibrate
out_cal = opj(temp_dir, '{}_cal'.format(master_burst_id))
cal_log = opj(out_dir, '{}_cal.err_log'.format(master_burst_id))
return_code = _calibration('{}.dim'.format(master_import), out_cal,
cal_log, product_type)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
if not coherence:
# remove imports
h.delete_dimap(master_import)
# speckle filtering
if speckle_filter:
speckle_import = opj(temp_dir,
'{}_speckle_import'.format(master_burst_id))
speckle_log = opj(out_dir,
'{}_speckle.err_log'.format(master_burst_id))
return_code = _speckle_filter('{}.dim'.format(out_cal), speckle_import,
speckle_log)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# remove temp file
h.delete_dimap(out_cal)
# reset master_import for follwoing routine
out_cal = speckle_import
# do terrain flattening in case it is selected
if product_type == 'RTC':
# define outfile
out_rtc = opj(temp_dir, '{}_rtc'.format(master_burst_id))
rtc_log = opj(out_dir, '{}_rtc.err_log'.format(master_burst_id))
# do the TF
return_code = _terrain_flattening('{}.dim'.format(out_cal), out_rtc,
rtc_log, dem)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# remove tmp files
h.delete_dimap(out_cal)
# set out_rtc to out_cal for further processing
out_cal = out_rtc
if to_db:
out_db = opj(temp_dir, '{}_cal_db'.format(master_burst_id))
db_log = opj(out_dir, '{}_cal_db.err_log'.format(master_burst_id))
return_code = _linear_to_db('{}.dim'.format(out_cal), out_db, db_log)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# remove tmp files
h.delete_dimap(out_cal)
# set out_cal to out_db for further processing
out_cal = out_db
# geo code backscatter products
out_tc = opj(temp_dir, '{}_BS'.format(master_burst_id))
tc_log = opj(out_dir, '{}_BS_tc.err_log'.format(master_burst_id))
return_code = _terrain_correction('{}.dim'.format(out_cal), out_tc, tc_log,
resolution, dem)
# last check on backscatter data
return_code = h.check_out_dimap(out_tc)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# we move backscatter to final destination
h.move_dimap(out_tc, opj(out_dir, '{}_BS'.format(master_burst_id)))
if ls_mask_create:
# create LS map
out_ls = opj(temp_dir, '{}_LS'.format(master_burst_id))
ls_log = opj(out_dir, '{}_LS.err_log'.format(master_burst_id))
return_code = _ls_mask('{}.dim'.format(out_cal), out_ls, ls_log,
resolution, dem)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# last check on ls data
return_code = h.check_out_dimap(out_ls, test_stats=False)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# move ls data to final destination
h.move_dimap(out_ls, opj(out_dir, '{}_LS'.format(master_burst_id)))
# remove calibrated files
h.delete_dimap(out_cal)
if coherence:
# import slave
slave_import = opj(temp_dir, '{}_import'.format(slave_burst_id))
import_log = opj(out_dir, '{}_import.err_log'.format(slave_burst_id))
return_code = _import(slave_file, slave_import, import_log, swath,
slave_burst_nr)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# co-registration
# filelist = ['{}.dim'.format(master_import),
# '{}.dim'.format(slave_import)]
# filelist = '\'{}\''.format(','.join(filelist))
out_coreg = opj(temp_dir, '{}_coreg'.format(master_burst_id))
coreg_log = opj(out_dir, '{}_coreg.err_log'.format(master_burst_id))
# return_code = _coreg2(filelist, out_coreg, coreg_log, dem)
return_code = _coreg2('{}.dim'.format(master_import),
'{}.dim'.format(slave_import), out_coreg,
coreg_log, dem)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# remove imports
h.delete_dimap(master_import)
if remove_slave_import is True:
h.delete_dimap(slave_import)
# calculate coherence and deburst
out_coh = opj(temp_dir, '{}_c'.format(master_burst_id))
coh_log = opj(out_dir, '{}_coh.err_log'.format(master_burst_id))
return_code = _coherence('{}.dim'.format(out_coreg), out_coh, coh_log)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# remove coreg tmp files
h.delete_dimap(out_coreg)
# geocode
out_tc = opj(temp_dir, '{}_coh'.format(master_burst_id))
tc_log = opj(out_dir, '{}_coh_tc.err_log'.format(master_burst_id))
return_code = _terrain_correction('{}.dim'.format(out_coh), out_tc,
tc_log, resolution, dem)
# last check on coherence data
return_code = h.check_out_dimap(out_tc)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# move to final destination
h.move_dimap(out_tc, opj(out_dir, '{}_coh'.format(master_burst_id)))
# remove tmp files
h.delete_dimap(out_coh)
# write file, so we know this burst has been succesfully processed
if return_code == 0:
check_file = opj(out_dir, '.processed')
with open(str(check_file), 'w') as file:
file.write('passed all tests \n')
else:
h.remove_folder_content(temp_dir)
h.remove_folder_content(out_dir)
return return_code
def ard_to_ts(list_of_files,
processing_dir,
temp_dir,
burst,
proc_file,
product,
pol,
ncores=os.cpu_count()):
if type(list_of_files) == str:
list_of_files = list_of_files.replace("'", '').strip('][').split(', ')
# get the burst directory
burst_dir = opj(processing_dir, burst)
# check routine if timeseries has already been processed
check_file = opj(burst_dir, 'Timeseries',
'.{}.{}.processed'.format(product, pol))
if os.path.isfile(check_file):
print(' INFO: Timeseries of {} for {} in {} polarisation already'
' processed'.format(burst, product, pol))
return
# load ard parameters
with open(proc_file, 'r') as ard_file:
ard_params = json.load(ard_file)['processing parameters']
ard = ard_params['single ARD']
ard_mt = ard_params['time-series ARD']
if ard_mt['remove mt speckle'] is True:
ard_mt_speck = ard_params['time-series ARD']['mt speckle filter']
# get the db scaling right
to_db = ard['to db']
if to_db or product != 'bs':
to_db = False
print('INFO: Not converting to dB for {}'.format(product))
else:
to_db = ard_mt['to db']
print('INFO: Converting to dB for {}'.format(product))
if ard['apply ls mask']:
extent = opj(burst_dir, '{}.extent.masked.shp'.format(burst))
else:
extent = opj(burst_dir, '{}.extent.shp'.format(burst))
# min max dict for stretching in case of 16 or 8 bit datatype
mm_dict = {
'bs': {
'min': -30,
'max': 5
},
'coh': {
'min': 0.000001,
'max': 1
},
'Alpha': {
'min': 0.000001,
'max': 90
},
'Anisotropy': {
'min': 0.000001,
'max': 1
},
'Entropy': {
'min': 0.000001,
'max': 1
}
}
stretch = pol if pol in ['Alpha', 'Anisotropy', 'Entropy'] else product
# define out_dir for stacking routine
out_dir = opj(processing_dir, '{}'.format(burst), 'Timeseries')
os.makedirs(out_dir, exist_ok=True)
# create namespaces
temp_stack = opj(temp_dir, '{}_{}_{}'.format(burst, product, pol))
out_stack = opj(temp_dir, '{}_{}_{}_mt'.format(burst, product, pol))
stack_log = opj(out_dir,
'{}_{}_{}_stack.err_log'.format(burst, product, pol))
# run stacking routines
# convert list of files readable for snap
list_of_files = '\'{}\''.format(','.join(list_of_files))
if pol in ['Alpha', 'Anisotropy', 'Entropy']:
print(
' INFO: Creating multi-temporal stack of images of burst/track {} for'
' the {} band of the polarimetric H-A-Alpha'
' decomposition.'.format(burst, pol))
create_stack(list_of_files, temp_stack, stack_log, pattern=pol)
else:
print(
' INFO: Creating multi-temporal stack of images of burst/track {} for'
' {} product in {} polarization.'.format(burst, product, pol))
create_stack(list_of_files, temp_stack, stack_log, polarisation=pol)
# run mt speckle filter
if ard_mt['remove mt speckle'] is True:
speckle_log = opj(
out_dir, '{}_{}_{}_mt_speckle.err_log'.format(burst, product, pol))
print(' INFO: Applying multi-temporal speckle filter')
mt_speckle_filter('{}.dim'.format(temp_stack),
out_stack,
speckle_log,
speckle_dict=ard_mt_speck,
ncores=ncores)
# remove tmp files
h.delete_dimap(temp_stack)
else:
out_stack = temp_stack
if product == 'coh':
# get slave and master Date
mstDates = [
datetime.datetime.strptime(
os.path.basename(x).split('_')[3].split('.')[0], '%d%b%Y')
for x in glob.glob(opj('{}.data'.format(out_stack), '*img'))
]
slvDates = [
datetime.datetime.strptime(
os.path.basename(x).split('_')[4].split('.')[0], '%d%b%Y')
for x in glob.glob(opj('{}.data'.format(out_stack), '*img'))
]
# sort them
mstDates.sort()
slvDates.sort()
# write them back to string for following loop
sortedMstDates = [
datetime.datetime.strftime(ts, "%d%b%Y") for ts in mstDates
]
sortedSlvDates = [
datetime.datetime.strftime(ts, "%d%b%Y") for ts in slvDates
]
i, outfiles = 1, []
for mst, slv in zip(sortedMstDates, sortedSlvDates):
inMst = datetime.datetime.strptime(mst, '%d%b%Y')
inSlv = datetime.datetime.strptime(slv, '%d%b%Y')
outMst = datetime.datetime.strftime(inMst, '%y%m%d')
outSlv = datetime.datetime.strftime(inSlv, '%y%m%d')
infile = glob.glob(
opj('{}.data'.format(out_stack),
'*{}*{}_{}*img'.format(pol, mst, slv)))[0]
outfile = opj(
out_dir,
'{:02d}.{}.{}.{}.{}.tif'.format(i, outMst, outSlv, product,
pol))
ras.mask_by_shape(infile,
outfile,
extent,
to_db=to_db,
datatype=ard_mt['dtype output'],
min_value=mm_dict[stretch]['min'],
max_value=mm_dict[stretch]['max'],
ndv=0.0,
description=True)
# add ot a list for subsequent vrt creation
outfiles.append(outfile)
i += 1
else:
# get the dates of the files
dates = [
datetime.datetime.strptime(x.split('_')[-1][:-4], '%d%b%Y')
for x in glob.glob(opj('{}.data'.format(out_stack), '*img'))
]
# sort them
dates.sort()
# write them back to string for following loop
sortedDates = [
datetime.datetime.strftime(ts, "%d%b%Y") for ts in dates
]
i, outfiles = 1, []
for date in sortedDates:
# restructure date to YYMMDD
inDate = datetime.datetime.strptime(date, '%d%b%Y')
outDate = datetime.datetime.strftime(inDate, '%y%m%d')
infile = glob.glob(
opj('{}.data'.format(out_stack),
'*{}*{}*img'.format(pol, date)))[0]
# create outfile
outfile = opj(
out_dir, '{:02d}.{}.{}.{}.tif'.format(i, outDate, product,
pol))
ras.mask_by_shape(infile,
outfile,
extent,
to_db=to_db,
datatype=ard_mt['dtype output'],
min_value=mm_dict[stretch]['min'],
max_value=mm_dict[stretch]['max'],
ndv=0.0)
# add ot a list for subsequent vrt creation
outfiles.append(outfile)
i += 1
for file in outfiles:
return_code = h.check_out_tiff(file)
if return_code != 0:
h.remove_folder_content(temp_dir)
os.remove(file)
return return_code
# write file, so we know this ts has been succesfully processed
if return_code == 0:
with open(str(check_file), 'w') as file:
file.write('passed all tests \n')
# build vrt of timeseries
vrt_options = gdal.BuildVRTOptions(srcNodata=0, separate=True)
gdal.BuildVRT(opj(out_dir, 'Timeseries.{}.{}.vrt'.format(product, pol)),
outfiles,
options=vrt_options)
# remove tmp files
h.delete_dimap(out_stack)
def combine_timeseries(processing_dir, config_dict, timescan=True):
# namespaces for folder
comb_dir = processing_dir.joinpath('combined')
if comb_dir.exists():
h.remove_folder_content(comb_dir)
tseries_dir = comb_dir.joinpath('Timeseries')
tseries_dir.mkdir(parents=True, exist_ok=True)
PRODUCT_LIST = [
'bs.HH', 'bs.VV', 'bs.HV', 'bs.VH', 'coh.VV', 'coh.VH', 'coh.HH',
'coh.HV', 'pol.Entropy', 'pol.Anisotropy', 'pol.Alpha'
]
out_files, iter_list = [], []
for product_type in PRODUCT_LIST:
filelist = list(
processing_dir.glob(f'*/Timeseries/*{product_type}.tif'))
if len(filelist) > 1:
datelist = sorted([file.name.split('.')[1] for file in filelist])
for i, date in enumerate(datelist):
file = list(
processing_dir.glob(
f'*/Timeseries/*{date}*{product_type}.tif'))
outfile = tseries_dir.joinpath(
f'{i+1:02d}.{date}.{product_type}.tif')
shutil.copy(file[0], str(outfile))
out_files.append(str(outfile))
vrt_options = gdal.BuildVRTOptions(srcNodata=0, separate=True)
out_vrt = str(
tseries_dir.joinpath(f'Timeseries.{product_type}.vrt'))
gdal.BuildVRT(str(out_vrt), out_files, options=vrt_options)
if timescan:
from ost.generic import timescan as ts
ard = config_dict['processing']['single_ARD']
ard_mt = config_dict['processing']['time-series_ARD']
ard_tscan = config_dict['processing']['time-scan_ARD']
# get the db scaling right
to_db = ard['to_db']
if ard['to_db'] or ard_mt['to_db']:
to_db = True
dtype_conversion = True if ard_mt[
'dtype_output'] != 'float32' else False
tscan_dir = comb_dir.joinpath('Timescan')
tscan_dir.mkdir(parents=True, exist_ok=True)
# get timeseries vrt
time_series = tseries_dir.joinpath(
f'Timeseries.{product_type}.vrt')
if not time_series.exists():
continue
# create a datelist for harmonics
scene_list = [
str(file)
for file in list(tseries_dir.glob(f'*{product_type}.tif'))
]
# create a datelist for harmonics calculation
datelist = []
for file in sorted(scene_list):
datelist.append(os.path.basename(file).split('.')[1])
# define timescan prefix
timescan_prefix = tscan_dir.joinpath(f'{product_type}')
iter_list.append([
time_series, timescan_prefix, ard_tscan['metrics'],
dtype_conversion, to_db, ard_tscan['remove_outliers'],
datelist
])
if timescan:
# now we run with godale, which works also with 1 worker
executor = Executor(executor=config_dict['executor_type'],
max_workers=config_dict['max_workers'])
# run timescan creation
out_dict = {'track': [], 'prefix': [], 'metrics': [], 'error': []}
for task in executor.as_completed(func=ts.gd_mt_metrics,
iterable=iter_list):
burst, prefix, metrics, error = task.result()
out_dict['track'].append(burst)
out_dict['prefix'].append(prefix)
out_dict['metrics'].append(metrics)
out_dict['error'].append(error)
create_tscan_vrt(tscan_dir, config_dict)
def grds_to_ards(self,
inventory_df,
timeseries=False,
timescan=False,
mosaic=False,
overwrite=False,
max_workers=1,
executor_type='billiard'):
self.config_dict['max_workers'] = max_workers
self.config_dict['executor_type'] = executor_type
# in case we start from scratch, delete all data
# within processing folder
if overwrite:
logger.info('Deleting processing folder to start from scratch')
h.remove_folder_content(self.processing_dir)
# --------------------------------------------
# 2 Check if within SRTM coverage
# set ellipsoid correction and force GTC production
# when outside SRTM
center_lat = loads(self.aoi).centroid.y
if float(center_lat) > 59 or float(center_lat) < -59:
logger.info(
'Scene is outside SRTM coverage. Snap will therefore use '
'the GETASSE30 DEM. Also consider to use a stereographic '
'projection. and project to NSIDC Sea Ice Polar '
'Stereographic North projection (EPSG 3413).')
epsg = input(
'Please type the EPSG you want to project the output data or '
'just press enter for keeping Lat/Lon coordinate system '
'(e.g. 3413 for NSIDC Sea Ice Polar Stereographic North '
'projection, or 3976 for NSIDC Sea Ice Polar Stereographic '
'South projection')
if not epsg:
epsg = 4326
self.ard_parameters['single_ARD']['dem']['dem_name'] = 'GETASSE30'
self.ard_parameters['single_ARD']['dem']['out_projection'] = int(
epsg)
# --------------------------------------------
# 3 subset determination
# we need a check function that checks
self.config_dict.update(subset=vec.set_subset(self.aoi, inventory_df))
# dump to json file
with open(self.config_file, 'w') as outfile:
json.dump(self.config_dict, outfile, indent=4)
# --------------------------------------------
# 4 Check ard parameters in case they have been updated,
# and write them to json file
self.update_ard_parameters()
# --------------------------------------------
# 1 delete data in case of previous runs
# delete data in temporary directory in case there is
# something left from aborted previous runs
h.remove_folder_content(self.config_dict['temp_dir'])
# --------------------------------------------
# 5 set resolution in degree
# self.center_lat = loads(self.aoi).centroid.y
# if float(self.center_lat) > 59 or float(self.center_lat) < -59:
# logger.info(
# 'Scene is outside SRTM coverage. Will use 30m #
# 'ASTER DEM instead.'
# )
# self.ard_parameters['dem'] = 'ASTER 1sec GDEM'
# --------------------------------------------
# 5 set resolution in degree
# the grd to ard batch routine
processing_df = grd_batch.grd_to_ard_batch(inventory_df,
self.config_file)
# time-series part
if timeseries or timescan:
grd_batch.ards_to_timeseries(inventory_df, self.config_file)
if timescan:
grd_batch.timeseries_to_timescan(inventory_df, self.config_file)
if mosaic and timeseries:
grd_batch.mosaic_timeseries(inventory_df, self.config_file)
# --------------------------------------------
# 9 mosaic the timescans
if mosaic and timescan:
grd_batch.mosaic_timescan(self.config_file)
return processing_df
def bursts_to_ards(self,
timeseries=False,
timescan=False,
mosaic=False,
overwrite=False):
"""Batch processing function for full burst pre-processing workflow
This function allows for the generation of the
:param timeseries: if True, Time-series will be generated for
each burst id
:type timeseries: bool, optional
:param timescan: if True, Timescans will be generated for each burst id
type: timescan: bool, optional
:param mosaic: if True, Mosaics will be generated from the
Time-Series/Timescans of each burst id
:type mosaic: bool, optional
:param overwrite: (if True, the processing folder will be
emptied
:type overwrite: bool, optional
:param max_workers: number of parallel burst
:type max_workers: int, default=1
processing jobs
:return:
"""
# --------------------------------------------
# 2 Check if within SRTM coverage
# set ellipsoid correction and force GTC production
# when outside SRTM
center_lat = loads(self.aoi).centroid.y
if float(center_lat) > 59 or float(center_lat) < -59:
logger.info('Scene is outside SRTM coverage. Snap will therefore '
'use the Earth\'s geoid model.')
self.ard_parameters['single_ARD']['dem'][
'dem_name'] = 'Aster 1sec GDEM'
# --------------------------------------------
# 3 subset determination
# we need a check function that checks
self.config_dict['subset'] = False
# This does not work at the moment, and maybe does not even make sense,
# since for the co-registration we would need a sufficient
# part of the image
# self.config_dict['subset'] = vec.set_subset(
# self.aoi, self.burst_inventory
# )
# --------------------------------------------
# 4 Check ard parameters in case they have been updated,
# and write them to json file
self.update_ard_parameters()
# --------------------------------------------
# 1 delete data from previous runnings
# delete data in temporary directory in case there is
# something left from previous runs
h.remove_folder_content(self.config_dict['temp_dir'])
# in case we strat from scratch, delete all data
# within processing folder
if overwrite:
logger.info('Deleting processing folder to start from scratch')
h.remove_folder_content(self.config_dict['processing_dir'])
# --------------------------------------------
# 5 set resolution to degree
# self.ard_parameters['resolution'] = h.resolution_in_degree(
# self.center_lat, self.ard_parameters['resolution'])
if self.config_dict['max_workers'] > 1:
self.pre_download_srtm()
# --------------------------------------------
# 6 run the burst to ard batch routine (up to 3 times if needed)
i = 1
while i < 4:
processed_bursts_df = burst_batch.bursts_to_ards(
self.burst_inventory, self.config_file)
if False in processed_bursts_df.error.isnull().tolist():
i += 1
else:
i = 5
# write processed df to file
processing_dir = Path(self.config_dict['processing_dir'])
processed_bursts_df.to_pickle(
processing_dir.joinpath('processed_bursts.pickle'))
# if not all have been processed, raise an error to avoid
# false time-series processing
if i == 4:
raise RuntimeError(
'Not all all bursts have been successfully processed')
# --------------------------------------------
# 6 run the timeseries creation
if timeseries or timescan:
tseries_df = burst_batch.ards_to_timeseries(
self.burst_inventory, self.config_file)
# --------------------------------------------
# 7 run the timescan creation
if timescan:
burst_batch.timeseries_to_timescan(self.burst_inventory,
self.config_file)
# --------------------------------------------
# 8 mosaic the time-series
if mosaic and timeseries:
burst_batch.mosaic_timeseries(self.burst_inventory,
self.config_file)
# --------------------------------------------
# 9 mosaic the timescans
if mosaic and timescan:
burst_batch.mosaic_timescan(self.burst_inventory, self.config_file)
def mosaic_timescan(burst_inventory, config_file):
"""
:param burst_inventory:
:param config_file:
:return:
"""
print(' -----------------------------------------------------------------')
logger.info('Mosaicking time-scan layers.')
print(' -----------------------------------------------------------------')
# -------------------------------------
# 1 load project config
with open(config_file, 'r') as ard_file:
config_dict = json.load(ard_file)
processing_dir = Path(config_dict['processing_dir'])
metrics = config_dict['processing']['time-scan_ARD']['metrics']
if 'harmonics' in metrics:
metrics.remove('harmonics')
metrics.extend(['amplitude', 'phase', 'residuals'])
if 'percentiles' in metrics:
metrics.remove('percentiles')
metrics.extend(['p95', 'p5'])
# create output folder
ts_dir = processing_dir.joinpath('Mosaic/Timescan')
ts_dir.mkdir(parents=True, exist_ok=True)
temp_mosaic = processing_dir.joinpath('Mosaic/temp')
temp_mosaic.mkdir(parents=True, exist_ok=True)
# -------------------------------------
# 2 create iterable
# loop through each product
iter_list, vrt_iter_list = [], []
for product, metric in itertools.product(PRODUCT_LIST, metrics):
for track in burst_inventory.Track.unique():
filelist = list(
processing_dir.glob(
f'[A,D]{track}_IW*/Timescan/*{product}.{metric}.tif'))
if not len(filelist) >= 1:
continue
temp_acq = temp_mosaic.joinpath(f'{track}.{product}.{metric}.tif')
if temp_acq:
iter_list.append(
[track, metric, product, temp_acq, config_file])
# now we run with godale, which works also with 1 worker
executor = Executor(executor=config_dict['executor_type'],
max_workers=config_dict['max_workers'])
# run vrt creation
for task in executor.as_completed(func=mosaic.gd_mosaic_slc_acquisition,
iterable=iter_list):
task.result()
iter_list, vrt_iter_list = [], []
for product, metric in itertools.product(PRODUCT_LIST, metrics):
list_of_files = list(temp_mosaic.glob(f'*{product}.{metric}.tif'))
if not list_of_files:
continue
# turn to OTB readable format
list_of_files = ' '.join([str(file) for file in list_of_files])
# create namespace for outfile
outfile = ts_dir.joinpath(f'{product}.{metric}.tif')
check_file = outfile.parent.joinpath(f'.{outfile.name[:-4]}.processed')
if check_file.exists():
logger.info(f'Mosaic layer {outfile.name} already processed.')
continue
logger.info(f'Mosaicking layer {outfile.name}.')
iter_list.append([list_of_files, outfile, config_file])
# now we run with godale, which works also with 1 worker
executor = Executor(executor=config_dict['executor_type'],
max_workers=config_dict['max_workers'])
# run mosaicking
for task in executor.as_completed(func=mosaic.gd_mosaic,
iterable=iter_list):
task.result()
ras.create_tscan_vrt(ts_dir, config_file)
# remove temp folder
h.remove_folder_content(temp_mosaic)
def mosaic_timeseries(burst_inventory, config_file):
print(' -----------------------------------------------------------------')
logger.info('Mosaicking time-series layers.')
print(' -----------------------------------------------------------------')
# -------------------------------------
# 1 load project config
with open(config_file, 'r') as ard_file:
config_dict = json.load(ard_file)
processing_dir = Path(config_dict['processing_dir'])
# create output folder
ts_dir = processing_dir.joinpath('Mosaic/Timeseries')
ts_dir.mkdir(parents=True, exist_ok=True)
temp_mosaic = processing_dir.joinpath('Mosaic/temp')
temp_mosaic.mkdir(parents=True, exist_ok=True)
# -------------------------------------
# 2 create iterable
# loop through each product
iter_list, vrt_iter_list = [], []
for product in PRODUCT_LIST:
for track in burst_inventory.Track.unique():
dates = [
date[2:] for date in sorted(burst_inventory.Date[
burst_inventory.Track == track].unique())
]
for i, date in enumerate(dates):
if 'coh' in product:
# we do the try, since for the last date
# there is no dates[i+1] for coherence
try:
temp_acq = temp_mosaic.joinpath(
f'{i}.{date}.{dates[i + 1]}.{track}.{product}.tif')
except IndexError:
temp_acq = None
else:
temp_acq = temp_mosaic.joinpath(
f'{i}.{date}.{track}.{product}.tif')
if temp_acq:
iter_list.append(
[track, date, product, temp_acq, config_file])
# now we run with godale, which works also with 1 worker
executor = Executor(executor=config_dict['executor_type'],
max_workers=config_dict['max_workers'])
# run vrt creation
for task in executor.as_completed(func=mosaic.gd_mosaic_slc_acquisition,
iterable=iter_list):
task.result()
# mosaic the acquisitions
iter_list, vrt_iter_list = [], []
for product in PRODUCT_LIST:
outfiles = []
for i in range(len(dates)):
list_of_files = list(temp_mosaic.glob(f'{i}.*{product}.tif'))
if not list_of_files:
continue
datelist = []
for file in list_of_files:
if 'coh' in product:
datelist.append(
f"{file.name.split('.')[2]}_{file.name.split('.')[1]}")
else:
datelist.append(file.name.split('.')[1])
# get start and endate of mosaic
start, end = sorted(datelist)[0], sorted(datelist)[-1]
list_of_files = ' '.join([str(file) for file in list_of_files])
# create namespace for output file
if start == end:
outfile = ts_dir.joinpath(f'{i + 1:02d}.{start}.{product}.tif')
# with the above operation, the list automatically
# turns into string, so we can call directly list_of_files
shutil.move(list_of_files, outfile)
outfiles.append(outfile)
continue
else:
outfile = ts_dir.joinpath(
f'{i + 1:02d}.{start}-{end}.{product}.tif')
# create namespace for check_file
check_file = outfile.parent.joinpath(
f'.{outfile.name[:-4]}.processed')
if check_file.exists():
logger.info(f'Mosaic layer {outfile} already processed.')
continue
# append to list of outfile for vrt creation
outfiles.append(outfile)
iter_list.append([list_of_files, outfile, config_file])
vrt_iter_list.append([ts_dir, product, outfiles])
# now we run with godale, which works also with 1 worker
executor = Executor(executor=config_dict['executor_type'],
max_workers=config_dict['max_workers'])
# run mosaicking
for task in executor.as_completed(func=mosaic.gd_mosaic,
iterable=iter_list):
task.result()
# run mosaicking vrts
for task in executor.as_completed(func=mosaic.create_timeseries_mosaic_vrt,
iterable=vrt_iter_list):
task.result()
# remove temp folder
h.remove_folder_content(temp_mosaic)
def burst_to_ard(master_file,
swath,
master_burst_nr,
master_burst_id,
proc_file,
out_dir,
temp_dir,
slave_file=None,
slave_burst_nr=None,
slave_burst_id=None,
coherence=False,
remove_slave_import=False,
ncores=os.cpu_count()):
'''The main routine to turn a burst into an ARD product
Args:
master_file (str): path to full master SLC scene
swath (str): subswath
master_burst_nr (): index number of the burst
master_burst_id ():
out_dir (str):
temp_dir (str):
slave_file (str):
slave_burst_nr (str):
slave_burst_id (str):
proc_file (str):
remove_slave_import (bool):
ncores (int): number of cpus used - useful for parallel processing
'''
if type(remove_slave_import) == str:
if remove_slave_import == 'True':
remove_slave_import = True
elif remove_slave_import == 'False':
remove_slave_import = False
if type(coherence) == str:
if coherence == 'True':
coherence = True
elif coherence == 'False':
coherence = False
# load ards
with open(proc_file, 'r') as ard_file:
ard_params = json.load(ard_file)['processing parameters']
ard = ard_params['single ARD']
# ---------------------------------------------------------------------
# 1 Import
# import master
master_import = opj(temp_dir, '{}_import'.format(master_burst_id))
if not os.path.exists('{}.dim'.format(master_import)):
import_log = opj(out_dir, '{}_import.err_log'.format(master_burst_id))
polars = ard['polarisation'].replace(' ', '')
return_code = slc._import(master_file, master_import, import_log,
swath, master_burst_nr, polars, ncores
)
if return_code != 0:
h.delete_dimap(master_import)
return return_code
imported = '{}.dim'.format(master_import)
# ---------------------------------------------------------------------
# 2 H-A-Alpha
if ard['H-A-Alpha']:
# create HAalpha file
out_haa = opj(temp_dir, '{}_h'.format(master_burst_id))
haa_log = opj(out_dir, '{}_haa.err_log'.format(master_burst_id))
return_code = slc._ha_alpha(imported,
out_haa, haa_log,
ard['remove pol speckle'],
ard['pol speckle filter'],
ncores
)
# delete files in case of error
if return_code != 0:
h.delete_dimap(out_haa)
h.delete_dimap(master_import)
return return_code
# geo code HAalpha
out_htc = opj(temp_dir, '{}_pol'.format(master_burst_id))
haa_tc_log = opj(out_dir, '{}_haa_tc.err_log'.format(
master_burst_id))
return_code = common._terrain_correction(
'{}.dim'.format(out_haa), out_htc, haa_tc_log,
ard['resolution'], ard['dem'], ncores
)
# remove HAalpha tmp files
h.delete_dimap(out_haa)
# last check on the output files
return_code = h.check_out_dimap(out_htc)
if return_code != 0:
h.delete_dimap(out_htc)
h.delete_dimap(master_import)
return return_code
# move to final destination
h.move_dimap(out_htc, opj(out_dir, '{}_pol'.format(master_burst_id)))
# ---------------------------------------------------------------------
# 3 Calibration
out_cal = opj(temp_dir, '{}_cal'.format(master_burst_id))
cal_log = opj(out_dir, '{}_cal.err_log'.format(master_burst_id))
return_code = slc._calibration(
imported, out_cal, cal_log, ard['product type'], ncores)
# delete output if command failed for some reason and return
if return_code != 0:
h.delete_dimap(out_cal)
h.delete_dimap(master_import)
return return_code
if not coherence:
# remove imports
h.delete_dimap(master_import)
# ---------------------------------------------------------------------
# 4 Speckle filtering
if ard['remove speckle']:
speckle_import = opj(
temp_dir, '{}_speckle_import'.format(master_burst_id)
)
speckle_log = opj(
out_dir, '{}_speckle.err_log'.format(master_burst_id)
)
return_code = common._speckle_filter(
'{}.dim'.format(out_cal), speckle_import, speckle_log,
ard['speckle filter'], ncores
)
# remove input
h.delete_dimap(out_cal)
# delete output if command failed for some reason and return
if return_code != 0:
h.delete_dimap(speckle_import)
h.delete_dimap(master_import)
return return_code
# reset master_import for follwoing routine
out_cal = speckle_import
# ---------------------------------------------------------------------
# 5 Terrain Flattening
if ard['product type'] == 'RTC-gamma0':
# define outfile
out_rtc = opj(temp_dir, '{}_rtc'.format(master_burst_id))
rtc_log = opj(out_dir, '{}_rtc.err_log'.format(
master_burst_id))
# do the TF
return_code = common._terrain_flattening(
'{}.dim'.format(out_cal), out_rtc, rtc_log, ard['dem'], ncores
)
# remove tmp files
h.delete_dimap(out_cal)
# delete output if command failed for some reason and return
if return_code != 0:
h.delete_dimap(out_rtc)
h.delete_dimap(master_import)
return return_code
# set out_rtc to out_cal for further processing
out_cal = out_rtc
# ---------------------------------------------------------------------
# 7 to dB scale
if ard['to db']:
out_db = opj(temp_dir, '{}_cal_db'.format(master_burst_id))
db_log = opj(out_dir, '{}_cal_db.err_log'.format(master_burst_id))
return_code = common._linear_to_db('{}.dim'.format(out_cal), out_db, db_log, ncores)
# remove tmp files
h.delete_dimap(out_cal)
# delete output if command failed for some reason and return
if return_code != 0:
h.delete_dimap(out_db)
h.delete_dimap(master_import)
return return_code
# set out_cal to out_db for further processing
out_cal = out_db
# ---------------------------------------------------------------------
# 8 Geocode backscatter
if ard['product type'] != "Coherence_only":
out_tc = opj(temp_dir, '{}_bs'.format(master_burst_id))
tc_log = opj(out_dir, '{}_bs_tc.err_log'.format(master_burst_id))
return_code = common._terrain_correction(
'{}.dim'.format(out_cal), out_tc, tc_log,
ard['resolution'], ard['dem'], ncores)
# last check on backscatter data
return_code = h.check_out_dimap(out_tc)
if return_code != 0:
h.delete_dimap(out_tc)
return return_code
# we move backscatter to final destination
h.move_dimap(out_tc, opj(out_dir, '{}_bs'.format(master_burst_id)))
# ---------------------------------------------------------------------
# 9 Layover/Shadow mask
if ard['create ls mask']:
out_ls = opj(temp_dir, '{}_LS'.format(master_burst_id))
ls_log = opj(out_dir, '{}_LS.err_log'.format(master_burst_id))
return_code = common._ls_mask('{}.dim'.format(out_cal), out_ls, ls_log,
ard['resolution'], ard['dem'], ncores)
if return_code != 0:
h.delete_dimap(out_ls)
return return_code
# last check on ls data
return_code = h.check_out_dimap(out_ls, test_stats=False)
if return_code != 0:
h.delete_dimap(out_ls)
return return_code
# move ls data to final destination
h.move_dimap(out_ls, opj(out_dir, '{}_LS'.format(master_burst_id)))
# remove calibrated files
if ard['product type'] != "Coherence_only":
h.delete_dimap(out_cal)
if coherence:
# import slave
slave_import = opj(temp_dir, '{}_import'.format(slave_burst_id))
import_log = opj(out_dir, '{}_import.err_log'.format(slave_burst_id))
polars = ard['polarisation'].replace(' ', '')
return_code = slc._import(
slave_file, slave_import, import_log, swath, slave_burst_nr,
polars, ncores
)
if return_code != 0:
h.remove_folder_content(temp_dir)
return return_code
# co-registration
#filelist = ['{}.dim'.format(master_import),
# '{}.dim'.format(slave_import)]
#filelist = '\'{}\''.format(','.join(filelist))
out_coreg = opj(temp_dir, '{}_coreg'.format(master_burst_id))
coreg_log = opj(out_dir, '{}_coreg.err_log'.format(master_burst_id))
# return_code = _coreg(filelist, out_coreg, coreg_log, dem)
return_code = slc._coreg2('{}.dim'.format(master_import),
'{}.dim'.format(slave_import),
out_coreg,
coreg_log, ard['dem'], ncores)
# remove imports
h.delete_dimap(master_import)
if remove_slave_import is True:
h.delete_dimap(slave_import)
# delete output if command failed for some reason and return
if return_code != 0:
h.delete_dimap(out_coreg)
h.delete_dimap(slave_import)
return return_code
# calculate coherence and deburst
out_coh = opj(temp_dir, '{}_c'.format(master_burst_id))
coh_log = opj(out_dir, '{}_coh.err_log'.format(master_burst_id))
coh_polars = ard['coherence bands'].replace(' ', '')
return_code = slc._coherence('{}.dim'.format(out_coreg),
out_coh, coh_log, coh_polars, ncores)
# remove coreg tmp files
h.delete_dimap(out_coreg)
# delete output if command failed for some reason and return
if return_code != 0:
h.delete_dimap(out_coh)
h.delete_dimap(slave_import)
return return_code
# geocode
out_tc = opj(temp_dir, '{}_coh'.format(master_burst_id))
tc_log = opj(out_dir, '{}_coh_tc.err_log'.format(master_burst_id))
return_code = common._terrain_correction(
'{}.dim'.format(out_coh), out_tc, tc_log,
ard['resolution'], ard['dem'], ncores)
# remove tmp files
h.delete_dimap(out_coh)
# delete output if command failed for some reason and return
if return_code != 0:
h.delete_dimap(out_tc)
h.delete_dimap(slave_import)
return return_code
# remove tmp files
h.delete_dimap(out_coh)
# delete output if command failed for some reason and return
if return_code != 0:
h.delete_dimap(out_tc)
h.delete_dimap(slave_import)
return return_code
# last check on coherence data
return_code = h.check_out_dimap(out_tc)
if return_code != 0:
h.delete_dimap(out_tc)
return return_code
# move to final destination
h.move_dimap(out_tc, opj(out_dir, '{}_coh'.format(master_burst_id)))
# write out check file for tracking that it is processed
with open(opj(out_dir, '.processed'), 'w') as file:
file.write('passed all tests \n')
return return_code