def s1_preprocessing(archive, maxdate, processdir, sensor, product, resolution,
recursive, polarization, acquisition_mode, verbose):
"""Preprocess Sentinel-1 scenes"""
from pyroSAR import Archive, identify
from pyroSAR.snap import geocode
from datetime import datetime
archive = Archive(archive)
if maxdate == None:
maxdate = datetime.now().strftime("%Y%m%dT%H%M%S")
selection_proc = archive.select(processdir=processdir,
recursive=recursive,
polarizations=polarization,
maxdate=maxdate,
sensor=sensor,
product=product,
acquisition_mode=acquisition_mode,
verbose=verbose)
archive.close()
print(selection_proc)
for scene in selection_proc:
geocode(infile=scene, outdir=processdir, tr=resolution, scaling='db')
click.echo("eo2cube s1_create_archive")
def test_export_extra(self, tmpdir, testdata):
scene = testdata['s1']
with pytest.raises(RuntimeError):
geocode(scene, str(tmpdir), test=True,
export_extra=['foobar'])
geocode(scene, str(tmpdir), test=True,
export_extra=['localIncidenceAngle'])
def geocode_and_stack(infile: Path,
outfile: Path,
temp_dir: Path = Path("tmp/"),
dem_type="AW3D30",
**kwargs):
"""Process one sentinel 1 zip file. The main preprocessing is done inside geocode.
In the future, this may be rewritten to allow more control over the processing graph"""
bbox = identify(str(infile)).bbox()
print("Downloading DEM files")
vrt = dem_autoload(geometries=[bbox],
demType=dem_type,
vrt=f"{dem_type}.vrt",
buffer=0.01)
print("Converting DEM Files")
dem_create(vrt, str(f"{dem_type}.tif"))
geocode(str(infile),
str(temp_dir),
externalDEMFile=str(f"{dem_type}.tif"),
**kwargs)
os.remove(f"{dem_type}.tif")
file_list = [
str(file) for file in temp_dir.iterdir() if file.suffix == ".tif"
]
print("Stacking GeoTiffs")
stack_geotiffs(file_list, str(outfile))
shutil.rmtree(temp_dir)
def test_geotype(self, tmpdir, testdata):
scene = testdata['s1']
with pytest.raises(RuntimeError):
geocode(scene, str(tmpdir), geocoding_type='foobar', test=True)
geocode(scene,
str(tmpdir),
test=True,
geocoding_type='SAR simulation cross correlation')
def test_shp(self, tmpdir, testdata):
scene = testdata['s1']
ext = {'xmin': 12, 'xmax': 13, 'ymin': 53, 'ymax': 54}
with bbox(ext, 4326) as new:
with pytest.raises(RuntimeError):
geocode(scene, str(tmpdir), shapefile=new, test=True)
with identify(scene).bbox() as box:
ext = box.extent
ext['xmax'] -= 1
with bbox(ext, 4326) as new:
geocode(scene, str(tmpdir), shapefile=new, test=True)
def get_tile(START, END, gjson, out):
# search database for matching archives
print('Querying database...')
footprint = geojson_to_wkt(read_geojson(gjson))
products = api.query(footprint,
ingestiondate=(START, END),
platformname='Sentinel-1',
producttype='GRD',
sensoroperationalmode='IW',
orbitdirection='ASCENDING',
polarisationmode='VH')
# download archive
print('Downloading archive...')
pmd = api.download_all(products, directory_path='./temp/')
fname = './temp/' + list(pmd[0].values())[0]['title'] + '.zip'
# unpack and ingest
print('Unpacking archive...')
scene = pyroSAR.identify(fname)
scene.unpack('./temp/', overwrite=True)
# geocode
print('Geocoding data...')
shp = vector.Vector(filename=gjson)
geocode(infile=scene,
outdir='./temp/',
tr=int(sys.argv[3]),
scaling='db',
removeS1ThermalNoise=True,
demResamplingMethod='BISINC_21_POINT_INTERPOLATION',
terrainFlattening=True,
allow_RES_OSV=True,
speckleFilter='Refined Lee',
shapefile=shp,
cleanup=True)
# save image
print('Copying image...')
smd = scene.scanMetadata()
iname = './temp/' + [
file for file in os.listdir('./temp/')
if '{}__{}___{}_{}_VH'.format(smd['sensor'], smd['acquisition_mode'],
smd['orbit'], smd['start']) in file
][0]
shutil.copy2(iname, out)
# done
print('Done.')
def test_sliceassembly(self, tmpdir, testdata):
scene1 = testdata['s1']
scene2 = testdata['s1_2']
wf = geocode([scene1, scene2], str(tmpdir), test=True, returnWF=True)
for n in range(1, 4):
groups = groupbyWorkers(wf, n=n)
split(wf, groups)
def test_geocode(tmpdir, testdata):
scene = testdata['s1']
geocode(scene, str(tmpdir), test=True)
xmlfile = finder(str(tmpdir), ['*.xml'])[0]
tree = Workflow(xmlfile)
nodes = tree.nodes()
assert is_consistent(nodes) is True
groups = groupbyWorkers(xmlfile, 2)
split(xmlfile, groups)
id = identify(scene)
basename = '{}_{}'.format(id.outname_base(), tree.suffix)
procdir = os.path.join(str(tmpdir), basename)
assert os.path.isdir(procdir)
tempdir = os.path.join(procdir, 'temp')
assert os.path.isdir(tempdir)
parts = finder(tempdir, ['*.xml'])
assert len(parts) == 4
def test_pol(self, tmpdir, testdata):
scene = testdata['s1']
with pytest.raises(RuntimeError):
geocode(scene, str(tmpdir), polarizations=1, test=True)
with pytest.raises(RuntimeError):
geocode(scene, str(tmpdir), polarizations='foobar', test=True)
geocode(scene, str(tmpdir), polarizations='VV', test=True)
def test_refarea(self, tmpdir, testdata):
scene = testdata['s1']
with pytest.raises(RuntimeError):
geocode(scene, str(tmpdir), terrainFlattening=True, refarea='beta0', test=True)
with pytest.raises(ValueError):
geocode(scene, str(tmpdir), terrainFlattening=False, refarea='foobar', test=True)
geocode(scene, str(tmpdir), terrainFlattening=True, refarea='gamma0', test=True)
def worker(sitename):
#######################################################################################
# setup general processing parameters
resolution = 20
#######################################################################################
# define the directories for writing temporary and final results
sitedir = os.path.join(maindir, sitename)
outdir = os.path.join(sitedir, 'proc_out')
#######################################################################################
# load the test site geometry into a vector object
sites = vector.Vector('/.../testsites.shp')
# query the test site by name; a column name 'Site_Name' must be saved in your shapefile
site = sites["Site_Name='{}'".format(sitename)]
#######################################################################################
# query the database for scenes to be processed
with Archive(dbfile) as archive:
selection_proc = archive.select(vectorobject=site,
processdir=outdir,
sensor=('S1A', 'S1B'),
product='GRD',
acquisition_mode='IW',
vv=1)
print('{0}: {1} scenes found for site {2}'.format(socket.gethostname(),
len(selection_proc),
sitename))
#######################################################################################
# call to processing utility
if len(selection_proc) > 1:
print('start processing')
for scene in selection_proc:
geocode(infile=scene, outdir=outdir, tr=resolution, scaling='db')
return len(selection_proc)
if os.path.isdir(outdir2)==False:
os.mkdir(outdir2)
"""
if 'Ardfern' in site:
site_shapefile = '/home/dmilodow/DataStore_DTM/STFC/DATA/EDINAAerial/%s_2015/%s_bbox_wgs84.shp' % (site,site)
else:
site_shapefile = '/home/dmilodow/DataStore_DTM/STFC/DATA/EDINAAerial/%s_2017/%s_bbox_wgs84.shp' % (site,site)
dem_file = '/home/dmilodow/DataStore_DTM/STFC/DATA/OStopo/%s/%s_DTM_OSgrid.tif' % (site,site)
# process with pyroSAR
for scene in scenes:
geocode(scene, outdir=outdir1, t_srs=crs_epsg, tr=resolution, scaling='dB',
polarizations='all', geocoding_type='Range-Doppler',
removeS1BorderNoise=True, removeS1BorderNoiseMethod='pyroSAR',
removeS1ThermalNoise=True, terrainFlattening=True,
externalDEMFile=dem_file, externalDEMNoDataValue=dem_ndv,
externalDEMApplyEGM=True, speckleFilter=False,
refarea='gamma0', shapefile=site_shapefile)
"""
geocode(scene, outdir=outdir2, t_srs=crs_epsg, tr=resolution, scaling='dB',
polarizations='all', geocoding_type='Range-Doppler',
removeS1BorderNoise=True, removeS1BorderNoiseMethod='pyroSAR',
removeS1ThermalNoise=True, terrainFlattening=True,
externalDEMFile=dem_file, externalDEMNoDataValue=dem_ndv,
externalDEMApplyEGM=True, speckleFilter='Refined Lee',
refarea='gamma0', shapefile=site_shapefile)
"""
# Regrid final rasters onto the same grid. For now, use a nearest neighbour
# interpolation
# construct paths and logger
raw_path = os.path.dirname(scene)
ard_path = raw_path.replace('raw', 'ard') + "/snap"
# check scene content and credentials
if True: # checkScene( scene ):
out_path = os.path.join(ard_path, args.product)
snap.geocode(
infile=scene,
outdir=out_path,
t_srs=32632,
tr=args.res,
shapefile='/data/S1_ARD/code/aoi/testsite_alps.shp',
cleanup=False,
export_extra=[
'incidenceAngleFromEllipsoid', 'localIncidenceAngle',
'projectedLocalIncidenceAngle', 'DEM'
],
groupsize=1)
else:
print('scene crosses anti-meridian: ' + scene)
else:
print('no scenes found: ' + args.path)
else:
print('file does not exist: ' + args.dem)
def test_offset(self, tmpdir, testdata):
scene = testdata['s1']
geocode(scene, str(tmpdir), offset=(100, 100, 0, 0), test=True)
def geocode(infile, outdir, shapefile, job_id, job_id_vsc="9999", externalDEMFile=None,
refarea='gamma0', terrainFlattening=True, tmp_dir=None, logdir=None):
start_time = time.time()
identifier = os.path.splitext(os.path.split(infile)[1])[0]
if not tmp_dir:
tmp_dir = "/tmp"
tmp_dir_snap = os.path.join(tmp_dir, str(uuid4()), identifier)
os.makedirs(tmp_dir_snap, exist_ok=True)
os.makedirs(outdir, exist_ok=True)
if not logdir:
logdir = outdir + "/logs"
os.makedirs(outdir + "/logs", exist_ok=True)
# Setup logger
json_format = logging.Formatter(f'{{"filepath": "{infile}", "job_id": "{job_id}", "job_id_vsc": "{job_id_vsc}", "time": "%(asctime)s", "level": "%(levelname)s", "message": "%(message)s"}}')
log_file = os.path.join(logdir, f"{os.path.basename(infile)}.jsonl")
logging.basicConfig(filename=log_file,
level=logging.DEBUG)
root = logging.getLogger()
hdlr = root.handlers[0]
hdlr.setFormatter(json_format)
if externalDEMFile:
logging.info(f"Starting cropping DEM {externalDEMFile}.")
# Crop DEM to bbox of infile
externalDEMFile = crop_DEM(infile, externalDEMFile, tmp_dir_snap)
logging.info(f"Finished cropping DEM {externalDEMFile}.")
epsg = 4326
sampling = 10 # in meters
# Direct SNAP/pyroSAR stdout to Python variable
try:
sys.stdout = mystdout = StringIO()
logging.info("Starting pyroSAR SNAP processing chain.")
xml_file = snap.geocode(infile=infile,
outdir=tmp_dir_snap,
refarea=refarea,
t_srs=epsg,
tr=sampling,
shapefile=shapefile,
externalDEMFile=externalDEMFile,
externalDEMApplyEGM=False,
terrainFlattening=terrainFlattening,
cleanup=True, allow_RES_OSV=True,
scaling='linear', groupsize=1, removeS1ThermalNoise=True,
returnWF=True
)
# Set stdout to default value
sys.stdout = sys.__stdout__
# Analyze logs form SNAP/pyroSAR
error_patterns = ('failed', 'error', 'exception', 'severe')
skip_patterns = ('\t', 'Caused by')
output_lines = mystdout.getvalue().split('\n')
equi7_flag = True
for output_line in output_lines:
skip_flag = any([True if pattern in output_line.lower() else False for pattern in skip_patterns]) or len(output_line) == 0
error_flag = any([True if pattern in output_line.lower() else False for pattern in error_patterns])
# if output_line.startswith('\t') or output_line.startswith('Caused by') or len(output_line) == 0:
if skip_flag:
# Skip embedded Java tracebacks or empty lines
continue
elif error_flag:
# if any([True if pattern in output_line.lower() else False for pattern in error_patterns]):
logging.error(output_line)
equi7_flag = False
else:
logging.info(output_line)
if equi7_flag:
logging.info("Finished pyroSAR SNAP processing chain.")
# Move log file and SNAP xml file to outdir
logging.info("Starting to tile output with Equi7.")
shutil.move(xml_file, xml_file.replace(tmp_dir_snap, outdir))
orb_dir = os.path.basename(f"{xml_file}")[10].replace('A', 'ascending').replace('D', 'descending')
e7 = equi7grid.Equi7Grid(sampling=10)
file_list = glob(tmp_dir_snap + "/S1*.tif")
for k, this_file in enumerate(file_list):
# Ad polarization info to snap filename
file_dir, filename = (os.path.dirname(this_file), os.path.basename(this_file))
filename = filename[:28] + identifier[9] + filename[28:]
this_file_new = os.path.join(file_dir, filename)
shutil.move(this_file, this_file_new)
this_file = this_file_new
# Add metadata to geotiff headers
subprocess.run(['gdal_edit.py',
'-mo', 'snap_version=7.0',
'-mo', f'snap_xml_graph={xml_file}',
'-mo', 'proc_facility=EODC-cloud',
'-mo', f'start_time={identifier[17:32]}',
'-mo', f'end_time={identifier[33:48]}',
'-mo', f'orbit_direction={orb_dir}',
this_file
], capture_output=True)
# Reproject and tile to EQUI7
_ = image2equi7grid(e7, image=this_file,
output_dir=outdir,
gdal_path="/usr/bin")
# Delete empty files (sometimes created by EQUI7)
output_files = glob(outdir + f"/EQUI7_EU010M/*/*{identifier[17:32]}*")
for output_file in output_files:
# Empty files created by EQUI7 are 734 KB in size
# Remove if filesize under 1 MB
filesize = round(os.path.getsize(output_file) / 1024)
if filesize < 1024:
os.remove(output_file)
logging.info(f"Deleted {output_file} with size {str(filesize)}K.")
shutil.rmtree(tmp_dir_snap)
logging.info(f"Deleted temp dir {tmp_dir_snap}")
logging.info("Finished to tile output with Equi7.")
proc_time = time.time() - start_time
minutes = floor(proc_time / 60)
seconds = round(((proc_time / 60) - minutes) * 60)
logging.info(f"Success! File {infile} processed in {str(minutes)} minutes and {str(seconds)} seconds.")
except Exception as exp:
# Set stdout to default value
sys.stdout = sys.__stdout__
logging.error(f"{exp}")
shutil.rmtree(tmp_dir_snap)
logging.info(f"Deleted temp dir {tmp_dir_snap}")
def test_scaling(self, tmpdir, testdata):
scene = testdata['s1']
with pytest.raises(RuntimeError):
geocode(scene, str(tmpdir), scaling='foobar', test=True)
def main(s1_archive,
out_path,
t_srs='utm',
tr=10,
polarization='all',
bbox=None,
shapefile=None,
scaling='db',
geocoding_type='Range-Doppler',
remove_s1_border_noise=True,
remove_s1_thermal_noise=False,
offset_left=None,
offset_right=None,
offset_top=None,
offset_bottom=None,
external_dem_file=None,
external_dem_no_data_value=None,
external_dem_apply_egm=True,
terrain_flattening=True,
basename_extension=None,
test=False,
export_extra=None,
group_size=2,
cleanup=True,
return_wf=False):
# identify the scene from input archive
print("Identifying S1 scene from input archive.")
scene = pyroSAR.identify(s1_archive)
print("Success")
# deal with offsets
offset = (offset_left, offset_right, offset_top, offset_bottom)
if set(offset) == set([None]):
offset = None
if t_srs.lower() == 'utm':
# get the bbox
scene_extent = scene.bbox().extent
scene_bbox = box(scene_extent['xmin'], scene_extent['ymin'],
scene_extent['xmax'], scene_extent['ymax'])
# get the lat and long from the aoi centroid
long = scene_bbox.centroid.x
lat = scene_bbox.centroid.y
# use the longitude to get the utm zone (3rd element in returned list
c = utm.from_latlon(lat, long)
utm_zone = c[2]
# build the epsg code
if lat >= 0.0:
t_srs = int("326%s" % (str(utm_zone)))
else:
t_srs = int("327%s" % (str(utm_zone)))
else:
# try to convert to an integer -- if it doesn't work, leave as is
try:
t_srs = int(t_srs)
except ValueError:
pass
# deal with basename extensions
if basename_extension is None:
basename_extensions = None
else:
basename_extensions = [basename_extension]
# deal with export_extra
if export_extra is not None:
export_extra = list(map(str, export_extra.split(',')))
# make the output directory if it doesn't exist
if os.path.exists(out_path) is False:
os.mkdir(out_path)
# if bbox is provided, convert to format expected by pyroSAR
if bbox is not None:
bbox = map(float, bbox.split(','))
shapefile = spatialist.bbox(
dict(zip(['xmin', 'ymin', 'xmax', 'ymax'], bbox)),
crs=spatialist.auxil.crsConvert(4326, 'epsg'))
print("Geocoding image according to input options.")
geocode(infile=scene,
outdir=out_path,
t_srs=t_srs,
tr=tr,
polarizations=[polarization],
shapefile=shapefile,
scaling=scaling,
geocoding_type=geocoding_type,
removeS1BoderNoise=remove_s1_border_noise,
removeS1ThermalNoise=remove_s1_thermal_noise,
offset=offset,
externalDEMFile=external_dem_file,
externalDEMNoDataValue=external_dem_no_data_value,
externalDEMApplyEGM=external_dem_apply_egm,
terrainFlattening=terrain_flattening,
basename_extensions=basename_extensions,
test=test,
export_extra=export_extra,
groupsize=group_size,
cleanup=cleanup,
returnWF=return_wf)
print("Processing completed successfully.")
def test_pol_list(self, tmpdir, testdata):
scene = testdata['s1']
geocode(scene, str(tmpdir), polarizations=['VV', 'VH'], test=True)
def test_srs(self, tmpdir, testdata):
scene = testdata['s1']
with pytest.raises(RuntimeError):
geocode(scene, str(tmpdir), t_srs='foobar', test=True)
geocode(scene, str(tmpdir), t_srs=32632, test=True)
def test_geocode():
scene = 'pyroSAR/tests/data/S1A_IW_GRDH_1SDV_20150222T170750_20150222T170815_004739_005DD8_3768.zip'
geocode(scene, 'pyroSAR/tests/data', test=True)
os.remove(
'pyroSAR/tests/data/S1A__IW___A_20150222T170750_bnr_Orb_Cal_TF_TC_dB_proc.xml'
)
def test_infile_type(self, tmpdir, testdata):
scene = testdata['s1']
with pytest.raises(TypeError):
geocode(infile=123, outdir=str(tmpdir), test=True)
id = identify(scene)
geocode(infile=id, outdir=str(tmpdir), test=True)
def test_geocode(tmpdir, testdata):
scene = testdata['s1']
geocode(scene, str(tmpdir), test=True)
def test_externalDEM(self, tmpdir, testdata):
scene = testdata['s1']
dem_dummy = testdata['tif']
with pytest.raises(RuntimeError):
geocode(scene, str(tmpdir), externalDEMFile='foobar', test=True)
geocode(scene, str(tmpdir), externalDEMFile=dem_dummy, test=True)
def test_speckleFilter(self, tmpdir, testdata):
scene = testdata['s1']
with pytest.raises(ValueError):
geocode(scene, str(tmpdir), speckleFilter='foobar', test=True)
geocode(scene, str(tmpdir), speckleFilter='Refined Lee', test=True)