def batch_download(inventory_df, download_dir, uname, pword, concurrent=2):
from ost import Sentinel1_Scene as S1Scene
from ost.helpers import onda
# create list of scenes
scenes = inventory_df['identifier'].tolist()
check, i = False, 1
while check is False and i <= 10:
download_list = []
for scene_id in scenes:
scene = S1Scene(scene_id)
filepath = scene._download_path(download_dir, True)
try:
uuid = (inventory_df['uuid'][inventory_df['identifier'] ==
scene_id].tolist())
except KeyError:
uuid = scene.ondadias_uuid(
opener=onda.connect(uname=uname, pword=pword))
if os.path.exists('{}.downloaded'.format(filepath)):
print(' INFO: {} is already downloaded.'.format(
scene.scene_id))
else:
# create list objects for download
download_list.append([uuid[0], filepath, uname, pword])
if download_list:
pool = multiprocessing.Pool(processes=concurrent)
pool.map(s1_download, download_list)
downloaded_scenes = glob.glob(
opj(download_dir, 'SAR', '*', '20*', '*', '*', '*.zip.downloaded'))
if len(inventory_df['identifier'].tolist()) == len(downloaded_scenes):
print(' INFO: All products are downloaded.')
check = True
else:
check = False
for scene in scenes:
scene = S1Scene(scene)
filepath = scene._download_path(download_dir)
if os.path.exists('{}.downloaded'.format(filepath)):
scenes.remove(scene.scene_id)
i += 1
def batch_download(inventory_df, download_dir, uname, pword, concurrent=10):
from ost import Sentinel1Scene as S1Scene
# create list with scene ids to download
scenes = inventory_df['identifier'].tolist()
asf_list = []
for scene_id in scenes:
# initialize scene instance and get destination filepath
scene = S1Scene(scene_id)
file_path = scene.download_path(download_dir, True)
# check if already downloaded
if file_path.with_suffix('.downloaded').exists():
logger.info(f'{scene.scene_id} has been already downloaded.')
continue
# append to list
asf_list.append([scene.asf_url(), file_path, uname, pword])
# if list is not empty, do parallel download
check_counter = 0
if asf_list:
executor = Executor(max_workers=concurrent,
executor='concurrent_processes')
for task in executor.as_completed(
func=asf_download_parallel,
iterable=asf_list,
fargs=[]
):
task.result()
check_counter += 1
# if all have been downloaded then we are through
if len(inventory_df['identifier'].tolist()) == check_counter:
logger.info('All products are downloaded.')
# else we
else:
for scene in scenes:
# we check if outputfile exists...
scene = S1Scene(scene)
file_path = scene.download_path(download_dir)
if file_path.with_suffix('.downloaded').exists():
# ...and remove from list of scenes to download
scenes.remove(scene.scene_id)
else:
raise DownloadError(
'ASF download is incomplete or has failed. Try to re-run.'
)
def batch_download(inventory_df, download_dir, uname, pword, concurrent=2):
from ost import Sentinel1Scene as S1Scene
from ost.helpers import onda
# create list of scenes
scenes = inventory_df['identifier'].tolist()
check, i = False, 1
while check is False and i <= 10:
download_list = []
for scene_id in scenes:
scene = S1Scene(scene_id)
file_path = scene.download_path(download_dir, True)
try:
uuid = (inventory_df['uuid'][inventory_df['identifier'] ==
scene_id].tolist())
except KeyError:
uuid = scene.ondadias_uuid(
opener=onda.connect(uname=uname, pword=pword))
if file_path.with_suffix('.downloaded').exists():
logger.info(f'{scene.scene_id} is already downloaded.')
else:
# create list objects for download
download_list.append([uuid[0], file_path, uname, pword])
if download_list:
pool = multiprocessing.Pool(processes=concurrent)
pool.map(onda_download, download_list)
downloaded_scenes = list(download_dir.glob('**/*.downloaded'))
if len(inventory_df['identifier'].tolist()) == len(downloaded_scenes):
logger.info('All products are downloaded.')
check = True
else:
check = False
for scene in scenes:
scene = S1Scene(scene)
file_path = scene.download_path(download_dir)
if file_path.with_suffix('.downloaded'):
scenes.remove(scene.scene_id)
i += 1
def _execute_batch_burst_ard(burst, processing_dir, download_dir, data_mount,
ard_parameters):
index, burst = burst
resolution = ard_parameters['resolution']
product_type = ard_parameters['product_type']
speckle_filter = ard_parameters['speckle_filter']
ls_mask_create = ard_parameters['ls_mask_create']
to_db = ard_parameters['to_db']
dem = ard_parameters['dem']
logger.debug('INFO: Entering burst {} at date {}.'.format(
burst.BurstNr, burst.Date))
master_scene = S1Scene(burst.SceneID)
# get path to file
master_file = master_scene.get_path(download_dir, data_mount)
# get subswath
subswath = burst.SwathID
# get burst number in file
master_burst_nr = burst.BurstNr
# create a fileId
master_id = '{}_{}'.format(burst.Date, burst.bid)
# create out folder
out_dir = '{}/{}/{}'.format(processing_dir, burst.bid, burst.Date)
os.makedirs(out_dir, exist_ok=True)
# check if already processed
if os.path.isfile(opj(out_dir, '.processed')):
logger.debug('INFO: Burst {} from {} already processed'.format(
burst.bid, burst.Date))
return_code = 0
return return_code
with TemporaryDirectory() as temp_dir:
try:
return_code = burst_to_ard.burst_to_ard(
master_file=master_file,
swath=subswath,
master_burst_nr=burst['BurstNr'],
master_burst_id=master_id,
master_burst_poly=burst['geometry'],
out_dir=out_dir,
out_prefix=master_id,
temp_dir=temp_dir,
resolution=resolution,
product_type=product_type,
speckle_filter=speckle_filter,
to_db=to_db,
ls_mask_create=ls_mask_create,
dem=dem,
)
except Exception as e:
raise e
if return_code != 0:
raise RuntimeError('Something went wrong with the GPT processing! '
'with return code: %s' % return_code)
return return_code
def _prepare_scenes_to_dl(inventory_df, download_dir, uname, pword):
logger.debug(
'INFO: Getting the storage status (online/onTape) of each scene.')
logger.debug('INFO: This may take a while.')
# this function does not just check,
# but it already triggers the production of the S1 scene
inventory_df['pepsStatus'], inventory_df['pepsUrl'] = (zip(*[
S1Scene(product).peps_online_status(uname, pword)
for product in inventory_df.identifier.tolist()
]))
# as long as there are any scenes left for downloading, loop
while len(inventory_df[inventory_df['pepsStatus'] != 'downloaded']) > 0:
# excluded downlaoded scenes
inventory_df = inventory_df[inventory_df['pepsStatus'] != 'downloaded']
# recheck for status
inventory_df['pepsStatus'], inventory_df['pepsUrl'] = (zip(*[
S1Scene(product).peps_online_status(uname, pword)
for product in inventory_df.identifier.tolist()
]))
# if all scenes to download are on Tape, we wait for a minute
if len(inventory_df[inventory_df['pepsStatus'] == 'online']) == 0:
logger.debug(
'INFO: Imagery still on tape, we will wait for 1 minute '
'and try again.')
time.sleep(60)
# create the peps_list for parallel download
download_list = []
for index, row in (
inventory_df[inventory_df['pepsStatus'] == 'online'].iterrows()):
# get scene identifier
scene_id = row.identifier
# construct download path
scene = S1Scene(scene_id)
download_path = scene._download_path(download_dir, True)
# put all info to the peps_list for parallelised download
download_list.append([
inventory_df.pepsUrl[inventory_df.identifier == scene_id].tolist()
[0], download_path, uname, pword
])
return download_list
def batch_download(inventory_df, download_dir, uname, pword, concurrent=10):
# create list of scenes
scenes = inventory_df['identifier'].tolist()
# print(scenes)
check, i = False, 1
while check is False and i <= 10:
asf_list = []
for scene_id in scenes:
scene = S1Scene(scene_id)
filepath = scene._download_path(download_dir, True)
if os.path.exists('{}.downloaded'.format(filepath)):
print(' INFO: {} is already downloaded.'
.format(scene.scene_id))
else:
asf_list.append([scene.asf_url(), filepath,
uname, pword])
if asf_list:
pool = multiprocessing.Pool(processes=concurrent)
pool.map(s1_download, asf_list)
downloaded_scenes = glob.glob(
opj(download_dir, 'SAR', '*', '20*', '*', '*',
'*.zip.downloaded'))
if len(inventory_df['identifier'].tolist()) == len(downloaded_scenes):
check = True
else:
check = False
for scene in scenes:
scene = S1Scene(scene)
filepath = scene._download_path(download_dir)
if os.path.exists('{}.downloaded'.format(filepath)):
scenes.remove(scene.scene_id)
i += 1
def _check_downloaded_files(inventory_df, download_dir):
# routine to check if the file has been downloaded
for index, row in (
inventory_df[inventory_df['pepsStatus'] == 'online'].iterrows()):
# get scene identifier
scene_id = row.identifier
# construct download path
scene = S1Scene(scene_id)
download_path = scene._download_path(download_dir)
if os.path.exists(download_path):
inventory_df.at[index, 'pepsStatus'] = 'downloaded'
class TestS1Scene(unittest.TestCase):
scene_id = ['S1A_IW_GRDH_1SDV_20191104T170638_20191104T170703_029764_036486_DB88']
s1 = S1Scene(self.scene_id)
def test_attributes(self):
self.assertIs(self.s1.scene_id, self.scene_id)
self.assertIs(self.s1.mission_id, 'S1A')
def test_paths(self):
def get_scene_path(scene_id):
scene = S1Scene(scene_id)
if scene.creodias_path():
path = scene.creodias_path()
elif scene.mundi_path():
path = scene.mundi_path()
elif scene.onda_path():
path = scene.onda_path()
elif scene.download_path():
path = scene.download_path()
return path
def _check_downloaded_files(inventory_df, download_dir, downloaded_scenes,
missing_scenes):
from ost import Sentinel1Scene as S1Scene
scenes = inventory_df['identifier'].tolist()
if len(inventory_df['identifier'].tolist()) == len(
downloaded_scenes) and not missing_scenes:
logger.debug('INFO: All products are downloaded.')
check = True
else:
check = False
for scene in scenes:
scene = S1Scene(scene)
filepath = scene._download_path(download_dir)
if os.path.exists('{}.downloaded'.format(filepath)):
scenes.remove(scene.scene_id)
return check
def _prepare_scenes_to_dl(inventory_df, download_dir, uname, pword):
from ost import Sentinel1Scene as S1Scene
scenes = inventory_df['identifier'].tolist()
download_list = []
for scene_id in scenes:
scene = S1Scene(scene_id)
filepath = scene._download_path(download_dir, True)
try:
uuid = (inventory_df['uuid'][inventory_df['identifier'] ==
scene_id].tolist())
except Exception as e:
logger.debug(e)
uuid = scene.scihub_uuid(
connect(base_url='https://scihub.copernicus.eu/apihub/',
uname=uname,
pword=pword))
if os.path.exists('{}.downloaded'.format(filepath)):
logger.debug('INFO: {} is already downloaded.'.format(
scene.scene_id))
else:
# Create list objects for download
download_list.append([uuid, filepath, uname, pword])
return download_list
def batch_download(inventory_df, download_dir, uname, pword, concurrent=10):
from ost import Sentinel1Scene as S1Scene
logger.info('Getting the storage status (online/onTape) of each scene.')
logger.info('This may take a while.')
# this function does not just check,
# but it already triggers the production of the S1 scene
inventory_df['pepsStatus'], inventory_df['pepsUrl'] = (zip(*[
S1Scene(product).peps_online_status(uname, pword)
for product in inventory_df.identifier.tolist()
]))
# as long as there are any scenes left for downloading, loop
while len(inventory_df[inventory_df['pepsStatus'] != 'downloaded']) > 0:
# excluded downlaoded scenes
inventory_df = inventory_df[inventory_df['pepsStatus'] != 'downloaded']
# recheck for status
inventory_df['pepsStatus'], inventory_df['pepsUrl'] = (zip(*[
S1Scene(product).peps_online_status(uname, pword)
for product in inventory_df.identifier.tolist()
]))
# if all scenes to download are on Tape, we wait for a minute
if len(inventory_df[inventory_df['pepsStatus'] == 'online']) == 0:
logger.info('Imagery still on tape, we will wait for 1 minute '
'and try again.')
time.sleep(60)
# else we start downloading
else:
# create the peps_list for parallel download
peps_list = []
for index, row in (inventory_df[inventory_df['pepsStatus'] ==
'online'].iterrows()):
# get scene identifier
scene_id = row.identifier
# construct download path
scene = S1Scene(scene_id)
download_path = scene.download_path(download_dir, True)
# put all info to the peps_list for parallelised download
peps_list.append([
inventory_df.pepsUrl[inventory_df.identifier ==
scene_id].tolist()[0], download_path,
uname, pword
])
# parallelised download
pool = multiprocessing.Pool(processes=concurrent)
pool.map(peps_download, peps_list)
# routine to check if the file has been downloaded
for index, row in (inventory_df[inventory_df['pepsStatus'] ==
'online'].iterrows()):
# get scene identifier
scene_id = row.identifier
# construct download path
scene = S1Scene(scene_id)
download_path = scene.download_path(download_dir)
if download_path.exists():
inventory_df.at[index, 'pepsStatus'] = 'downloaded'
def burst_inventory(inventory_df, outfile, download_dir=os.getenv('HOME'),
data_mount='/eodata', uname=None, pword=None):
'''Creates a Burst GeoDataFrame from an OST inventory file
Args:
Returns:
'''
# create column names for empty data frame
column_names = ['SceneID', 'Track', 'Direction', 'Date', 'SwathID',
'AnxTime', 'BurstNr', 'geometry']
# crs for empty dataframe
crs = {'init': 'epsg:4326', 'no_defs': True}
# create empty dataframe
gdf_full = gpd.GeoDataFrame(columns=column_names, crs=crs)
# uname, pword = scihub.askScihubCreds()
for scene_id in inventory_df.identifier:
# read into S1scene class
scene = S1Scene(scene_id)
print(' INFO: Getting burst info from {}.'.format(scene.scene_id))
# get orbit direction
orbit_direction = inventory_df[
inventory_df.identifier == scene_id].orbitdirection.values[0]
filepath = scene.get_path(download_dir, data_mount)
if not filepath:
print(' INFO: Retrieving burst info from scihub'
' (need to download xml files)')
if not uname and not pword:
uname, pword = scihub.ask_credentials()
opener = scihub.connect(uname=uname, pword=pword)
if scene.scihub_online_status(opener) is False:
print(' INFO: Product needs to be online'
' to create a burst database.')
print(' INFO: Download the product first and '
' do the burst list from the local data.')
else:
single_gdf = scene._scihub_annotation_get(uname, pword)
elif filepath[-4:] == '.zip':
single_gdf = scene._zip_annotation_get(download_dir, data_mount)
elif filepath[-5:] == '.SAFE':
single_gdf = scene._safe_annotation_get(download_dir, data_mount)
# add orbit direction
single_gdf['Direction'] = orbit_direction
# append
gdf_full = gdf_full.append(single_gdf)
gdf_full = gdf_full.reset_index(drop=True)
for i in gdf_full['AnxTime'].unique():
# get similar burst times
idx = gdf_full.index[(gdf_full.AnxTime >= i - 1) &
(gdf_full.AnxTime <= i + 1) &
(gdf_full.AnxTime != i)].unique().values
# reset all to first value
for j in idx:
gdf_full.at[j, 'AnxTime'] = i
# create the acrual burst id
gdf_full['bid'] = gdf_full.Direction.str[0] + \
gdf_full.Track.astype(str) + '_' + \
gdf_full.SwathID.astype(str) + '_' + \
gdf_full.AnxTime.astype(str)
# save file to out
gdf_full['Date'] = gdf_full['Date'].astype(str)
gdf_full['BurstNr'] = gdf_full['BurstNr'].astype(str)
gdf_full['AnxTime'] = gdf_full['AnxTime'].astype(str)
gdf_full['Track'] = gdf_full['Track'].astype(str)
gdf_full.to_file(outfile)
return gdf_full
def burst_to_ard_batch(burst_inventory, download_dir, processing_dir, temp_dir,
ard_parameters):
'''
'''
resolution = ard_parameters['resolution']
border_noise = ard_parameters['border_noise']
prdType = ard_parameters['product_type']
spkFlt = ard_parameters['speckle_filter']
ls_mask = ard_parameters['ls_mask']
to_db = ard_parameters['to_db']
dem = ard_parameters['dem']
coherence = ard_parameters['coherence']
polarimetry = ard_parameters['polarimetry']
for burst in burst_inventory.bid.unique():
# create a list of dates over which we loop
dates = burst_inventory.Date[burst_inventory.bid ==
burst].sort_values().tolist()
# loop through dates
for idx, date in enumerate(dates):
# get master date
master_date = dates[idx]
end = False
# try to get slave date
try:
slave_date = dates[
idx + 1] # here we will jave problems for the last one
except:
print(' Reached the end of the time-series')
# read master burst
master_burst = burst_inventory[
(burst_inventory.Date == master_date)
& (burst_inventory.bid == burst)]
master_scene = S1Scene(master_burst.SceneID.values[0])
# get path to file
master_file = master_scene.get_scene_path()
# get subswath
subswath = master_burst.SwathID.values[0]
# get burst number in file
master_burst_nr = master_burst.BurstNr.values[0]
# create a fileId
master_id = '{}_{}'.format(master_date, master_burst.bid.values[0])
# create logFile
logFile = '{}.errLog'.format(master_id)
# create out folder
out = '{}/{}/{}'.format(processing_dir, burst, date)
os.makedirs(out, exist_ok=True)
if end is True:
coherence = False
else:
# read slave burst
slave_burst = burst_inventory[
(burst_inventory.Date == slave_date)
& (burst_inventory.bid == burst)]
slave_scene = S1Scene(slave_burst.SceneID.values[0])
# get path to slave file
slave_file = slave_scene.get_scene_path()
# burst number in slave file (subswath is same)
slave_burst_nr = slave_burst.BurstNr.values[0]
# outFile name
slave_id = '{}_{}'.format(slave_date,
slave_burst.bid.values[0])
# run routine
burst2Ard.slcBurst2CohPolArd(master_file, slave_file, logFile,
subswath, master_burst_nr,
slave_burst_nr, out, master_id,
slave_id, temp_dir, prdType,
resolution)
def burst_to_ard_batch(burst_inventory,
download_dir,
processing_dir,
temp_dir,
ard_parameters,
data_mount='/eodata'):
'''Handles the batch processing of a OST complinat burst inventory file
Args:
burst_inventory (GeoDataFrame):
download_dir (str):
processing_dir (str):
temp_dir (str):
ard_parameters (dict):
'''
resolution = ard_parameters['resolution']
# border_noise = ard_parameters['border_noise']
product_type = ard_parameters['product_type']
speckle_filter = ard_parameters['speckle_filter']
ls_mask_create = ard_parameters['ls_mask_create']
to_db = ard_parameters['to_db']
dem = ard_parameters['dem']
coherence = ard_parameters['coherence']
polarimetry = ard_parameters['polarimetry']
pol_speckle_filter = ard_parameters['pol_speckle_filter']
for burst in burst_inventory.bid.unique(): # ***
# create a list of dates over which we loop
dates = burst_inventory.Date[burst_inventory.bid ==
burst].sort_values().tolist()
# loop through dates
for idx, date in enumerate(dates): # ******
print(' INFO: Entering burst {} at date {}.'.format(burst, date))
# get master date
master_date = dates[idx]
# we set this for handling the end of the time-series
end = False
coherence = ard_parameters['coherence']
# try to get slave date
try:
slave_date = dates[idx + 1] # last burst in timeseries?
except IndexError:
end = True
print(' INFO: Reached the end of the time-series.'
' Therefore no coherence calculation is done.')
else:
end = False
# read master burst
master_burst = burst_inventory[
(burst_inventory.Date == master_date)
& (burst_inventory.bid == burst)]
master_scene = S1Scene(master_burst.SceneID.values[0])
# get path to file
master_file = master_scene.get_path(download_dir, data_mount)
# get subswath
subswath = master_burst.SwathID.values[0]
# get burst number in file
master_burst_nr = master_burst.BurstNr.values[0]
# create a fileId
master_id = '{}_{}'.format(master_date, master_burst.bid.values[0])
# create out folder
out_dir = '{}/{}/{}'.format(processing_dir, burst, date)
os.makedirs(out_dir, exist_ok=True)
# check if already processed
if os.path.isfile(opj(out_dir, '.processed')):
print(' INFO: Burst {} from {} already processed'.format(
burst, date))
# return_code = 0
else:
if end is True:
coherence = False
slave_file, slave_burst_nr, slave_id = None, None, None
else:
# read slave burst
slave_burst = burst_inventory[
(burst_inventory.Date == slave_date)
& (burst_inventory.bid == burst)]
slave_scene = S1Scene(slave_burst.SceneID.values[0])
# get path to slave file
slave_file = slave_scene.get_path(download_dir, data_mount)
# burst number in slave file (subswath is same)
slave_burst_nr = slave_burst.BurstNr.values[0]
# outfile name
slave_id = '{}_{}'.format(slave_date,
slave_burst.bid.values[0])
# run routine
burst_to_ard.burst_to_ard(
master_file=master_file,
swath=subswath,
master_burst_nr=master_burst_nr,
master_burst_id=master_id,
out_dir=out_dir,
temp_dir=temp_dir,
slave_file=slave_file,
slave_burst_nr=slave_burst_nr,
slave_burst_id=slave_id,
coherence=coherence,
polarimetry=polarimetry,
pol_speckle_filter=pol_speckle_filter,
resolution=resolution,
product_type=product_type,
speckle_filter=speckle_filter,
to_db=to_db,
ls_mask_create=ls_mask_create,
dem=dem,
remove_slave_import=False)
def burst_inventory(inventory_df,
download_dir=os.getenv('HOME'),
uname=None,
pword=None):
'''Creates a Burst GeoDataFrame from an OST inventory file
Args:
Returns:
'''
# create column names for empty data frame
column_names = [
'SceneID', 'Track', 'Direction', 'Date', 'SwathID', 'AnxTime',
'BurstNr', 'geometry'
]
# crs for empty dataframe
crs = {'init': 'epsg:4326', 'no_defs': True}
# create empty dataframe
gdf_full = gpd.GeoDataFrame(columns=column_names, crs=crs)
# uname, pword = scihub.askScihubCreds()
for scene_id in inventory_df.identifier:
# read into S1scene class
scene = S1Scene(scene_id)
# get orbit direction
orbit_direction = inventory_df[inventory_df.identifier ==
scene_id].orbitdirection.values[0]
# check different data storages
if os.path.exists(scene.download_path(download_dir)):
print(' Getting burst info from downloaded files')
single_gdf = scene.download_annotation_get(download_dir)
elif os.path.exists(scene.creodias_path()):
print(' Getting burst info from Creodias eodata store')
single_gdf = scene.creodias_annotation_get()
else:
uname, pword = scihub.ask_credentials()
opener = scihub.connect(uname=uname, pword=pword)
print(' Getting burst info from scihub'
' (need to download xml files)')
if scene.scihub_online_status(opener) is False:
print(' INFO: Product needs to be online'
' to create a burst database.')
print(' INFO: Download the product first and '
' do the burst list from the local data.')
else:
single_gdf = scene.scihub_annotation_get(uname, pword)
# add orbit direction
single_gdf['Direction'] = orbit_direction
# append
gdf_full = gdf_full.append(single_gdf)
gdf_full = gdf_full.reset_index(drop=True)
for i in gdf_full['AnxTime'].unique():
# get similar burst times
idx = gdf_full.index[(gdf_full.AnxTime >= i - 1)
& (gdf_full.AnxTime <= i + 1) &
(gdf_full.AnxTime != i)].unique().values
# reset all to first value
for j in idx:
gdf_full.at[j, 'AnxTime'] = i
# create the acrual burst id
gdf_full['bid'] = gdf_full.Direction.str[0] + \
gdf_full.Track.astype(str) + '_' + \
gdf_full.SwathID.astype(str) + '_' + \
gdf_full.AnxTime.astype(str)
return gdf_full
def batch_download(inventory_df,
download_dir,
uname,
pword,
concurrent=2,
base_url='https://scihub.copernicus.eu/apihub'):
"""Batch download Sentinel-1 on the basis of an OST inventory GeoDataFrame
:param inventory_df:
:param download_dir:
:param uname:
:param pword:
:param concurrent:
:param base_url:
:return:
"""
from ost import Sentinel1Scene as S1Scene
from ost.helpers import scihub
if isinstance(download_dir, str):
download_dir = Path(download_dir)
# create list of scenes
scenes = inventory_df['identifier'].tolist()
check, i = False, 1
while check is False and i <= 10:
download_list = []
for scene_id in scenes:
scene = S1Scene(scene_id)
filepath = scene.download_path(download_dir, True)
try:
uuid = (inventory_df['uuid'][inventory_df['identifier'] ==
scene_id].tolist())
except KeyError:
uuid = [
scene.scihub_uuid(
connect(uname=uname, pword=pword, base_url=base_url))
]
if Path(f'{filepath}.downloaded').exists():
logger.debug(f'{scene.scene_id} is already downloaded.')
else:
# create list objects for download
download_list.append(
[uuid[0], filepath, uname, pword, base_url])
if download_list:
pool = multiprocessing.Pool(processes=concurrent)
pool.map(s1_download_parallel, download_list)
downloaded_scenes = list(download_dir.glob('**/*.downloaded'))
if len(inventory_df['identifier'].tolist()) == len(downloaded_scenes):
logger.info('All products are downloaded.')
check = True
else:
check = False
for scene in scenes:
scene = S1Scene(scene)
file_path = scene.download_path(download_dir)
if file_path.with_suffix('.downloaded').exists():
scenes.remove(scene.scene_id)
i += 1
def burst_to_ard_batch(burst_inventory, download_dir, processing_dir,
temp_dir, proc_file, data_mount='/eodata',
exec_file=None):
'''Handles the batch processing of a OST complinat burst inventory file
Args:
burst_inventory (GeoDataFrame):
download_dir (str):
processing_dir (str):
temp_dir (str):
ard_parameters (dict):
'''
# load ard parameters
with open(proc_file, 'r') as ard_file:
ard_params = json.load(ard_file)['processing parameters']
ard = ard_params['single ARD']
for burst in burst_inventory.bid.unique(): # ***
# create a list of dates over which we loop
dates = burst_inventory.Date[
burst_inventory.bid == burst].sort_values().tolist()
# loop through dates
for idx, date in enumerate(dates): # ******
print(' INFO: Entering burst {} at date {}.'.format(burst, date))
# get master date
master_date = dates[idx]
# we set this for handling the end of the time-series
end = False
coherence = ard['coherence']
# try to get slave date
try:
slave_date = dates[idx + 1] # last burst in timeseries?
except IndexError:
end = True
print(' INFO: Reached the end of the time-series.'
' Therefore no coherence calculation is done.')
else:
end = False
# read master burst
master_burst = burst_inventory[
(burst_inventory.Date == master_date) &
(burst_inventory.bid == burst)]
master_scene = S1Scene(master_burst.SceneID.values[0])
# get path to file
master_file = master_scene.get_path(download_dir, data_mount)
# get subswath
subswath = master_burst.SwathID.values[0]
# get burst number in file
master_burst_nr = master_burst.BurstNr.values[0]
# create a fileId
master_id = '{}_{}'.format(master_date, master_burst.bid.values[0])
# create out folder
out_dir = opj(processing_dir, burst, date)
os.makedirs(out_dir, exist_ok=True)
# check if already processed
if os.path.isfile(opj(out_dir, '.processed')):
print(' INFO: Burst {} from {} already processed'.format(
burst, date))
else:
if end is True:
coherence = False
slave_file, slave_burst_nr, slave_id = None, None, None
else:
# read slave burst
slave_burst = burst_inventory[
(burst_inventory.Date == slave_date) &
(burst_inventory.bid == burst)]
slave_scene = S1Scene(slave_burst.SceneID.values[0])
# get path to slave file
slave_file = slave_scene.get_path(download_dir,
data_mount)
# burst number in slave file (subswath is same)
slave_burst_nr = slave_burst.BurstNr.values[0]
# outfile name
slave_id = '{}_{}'.format(slave_date,
slave_burst.bid.values[0])
# just write command into a textfile
if exec_file:
# remove older files in case they exist
if os.path.isfile(exec_file):
os.remove(exec_file)
# construct command arguments
args = ('-m {} -ms {} -mn {} -mi {} -p {} -o {} -t {} '
'-s {} -sn {} -si {} -c {} -r {}').format(
master_file, subswath, master_burst_nr, master_id,
proc_file, out_dir, temp_dir,
slave_file, slave_burst_nr, slave_id,
coherence, False)
# get path to graph
rootpath = imp.find_module('ost')[1]
python_exe = opj(rootpath, 's1', 'burst_to_ard.py')
with open(exec_file, 'a') as exe:
exe.write('{} {} \n'.format(python_exe, args))
# run the command
else:
# run routine
burst_to_ard.burst_to_ard(
master_file=master_file,
swath=subswath,
master_burst_nr=master_burst_nr,
master_burst_id=master_id,
proc_file=proc_file,
out_dir=out_dir,
temp_dir=temp_dir,
slave_file=slave_file,
slave_burst_nr=slave_burst_nr,
slave_burst_id=slave_id,
coherence=coherence,
remove_slave_import=False)