def setUpClass(cls):
cls.product_id = 'S2A_MSIL1C_20180113T001101_N0206_R073_T55KGP_20180113T013328.SAFE'
metafiles = 'inspire '
cls.request_without_folder = AwsProductRequest(
bands='',
metafiles=metafiles,
safe_format=True,
product_id=cls.product_id)
cls.request_with_folder = AwsProductRequest(
data_folder=cls.OUTPUT_FOLDER,
bands='',
metafiles=metafiles,
safe_format=True,
product_id=cls.product_id)
def setUpClass(cls):
super().setUpClass()
cls.request = AwsProductRequest(data_folder=cls.OUTPUT_FOLDER,
metafiles='metadata,tileInfo,productInfo, datastrip/*/metadata',
product_id='S2A_MSIL2A_20180402T151801_N0207_R068_T33XWJ_20180402T202222')
cls.data = cls.request.get_data(save_data=True, redownload=True, data_filter=[20])
def setUpClass(cls):
super().setUpClass()
cls.request = AwsProductRequest(data_folder=cls.OUTPUT_FOLDER, bands='B10',
metafiles='metadata,tileInfo,productInfo, datastrip/*/metadata',
product_id='S2A_OPER_PRD_MSIL1C_PDMC_20160121T043931_R069_V20160103T171947_'
'20160103T171947')
cls.data = cls.request.get_data(save_data=True, redownload=True, max_threads=100)
def download_from_aws(scene_id: str, destination: str):
"""Download the Sentinel Scene from AWS.
It uses the library `sentinelhub-py <https://sentinelhub-py.readthedocs.io>`_ to download
the Sentinel-2 SAFE folder. Once downloaded, it compressed into a `zip`.
Notes:
Make sure to set both `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in environment variable.
This method does not raise Exception.
Args:
scene_id - Sentinel-2 Product Id (We call as scene_id)
destination - Path to store data. We recommend to use python `tempfile.TemporaryDirectory` and then move.
Returns:
Path to the downloaded file when success or None when an error occurred.
"""
try:
config = SHConfig()
config.aws_access_key_id = Config.AWS_ACCESS_KEY_ID
config.aws_secret_access_key = Config.AWS_SECRET_ACCESS_KEY
logging.info(f'Downloading {scene_id} From AWS...')
request = AwsProductRequest(product_id=scene_id,
data_folder=destination,
safe_format=True,
config=config)
_ = request.get_data(save_data=True)
file_name = '{}.SAFE'.format(scene_id)
logging.info(f'Compressing {scene_id}.SAFE...')
with working_directory(destination):
shutil.make_archive(base_dir=file_name,
format='zip',
base_name=scene_id)
return Path(destination) / file_name
except BaseException as e:
logging.error(f'Error downloading from AWS. {scene_id} - {str(e)}')
return None
def main(image_id,
out_path,
aws_access_key_id=None,
aws_secret_access_key=None,
aws_session_token=None,
tiles=None,
bands=None):
"""Download data from AWS open data and convert into SAFE format compatible with ESA tools"""
# set environment variables to enable AWS access
os.environ['AWS_ACCESS_KEY_ID'] = aws_access_key_id
os.environ['AWS_SECRET_ACCESS_KEY'] = aws_secret_access_key
os.environ['AWS_SESSION_TOKEN'] = aws_session_token
# parse input args
if tiles is not None:
tile_list = map(str, tiles.split(','))
else:
tile_list = None
if bands is not None:
band_list = map(str, bands.split(','))
else:
band_list = None
print("Compiling SAFE archive for {}...".format(image_id))
product_request = AwsProductRequest(product_id=image_id,
data_folder=out_path,
safe_format=True,
tile_list=tile_list,
bands=band_list)
product_request.save_data()
# zip it all up
# necessary because otherwise gbdx will drop empty folders and sen2cor will fail
print("Zipping up SAFE archive")
archive = os.path.join(out_path, '{}.SAFE'.format(image_id))
# use sh to do this because the shutil.make_archive() function doesn't seem to work with SNAP
sh.zip('-rm',
archive.replace('.SAFE', '.zip'),
os.path.basename(archive),
'-4',
_cwd=os.path.dirname(archive))
print("Process completed successfully.")
def setUpClass(cls):
super().setUpClass()
bands = 'B12'
metafiles = 'manifest,preview/B02'
tile = '1WCV'
cls.request = AwsProductRequest(data_folder=cls.OUTPUT_FOLDER, bands=bands,
metafiles=metafiles, tile_list=[tile],
product_id='S2A_MSIL1C_20171010T003621_N0205_R002_T01WCV_20171010T003615')
cls.data = cls.request.get_data(save_data=True, redownload=True, data_filter=[1])
def setUpClass(cls):
super().setUpClass()
cls.test_cases = [cls.SafeTestCase('L1C_02.01',
AwsProductRequest('S2A_OPER_PRD_MSIL1C_PDMC_20151218T020842_R115_V20151217T2'
'24602_20151217T224602', bands=AwsConstants.S2_L1C_BANDS,
metafiles=AwsConstants.S2_L1C_METAFILES,
tile_list=['T59HNA'], safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase('L1C_02.01_tile',
AwsTileRequest(tile='29KQB', time='2016-04-12', aws_index=None,
data_source=DataSource.SENTINEL2_L1C, safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase('L1C_02.02',
AwsProductRequest('S2A_OPER_PRD_MSIL1C_PDMC_20160606T232310_R121_V20160526T0'
'84351_20160526T084351.SAFE', tile_list=['34HCF'],
safe_format=True, data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase('L1C_02.04_old',
AwsProductRequest('S2A_OPER_PRD_MSIL1C_PDMC_20160910T174323_R071_V20160701T2'
'04642_20160701T204643', safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase('L1C_02.04',
AwsProductRequest('S2A_MSIL1C_20170413T104021_N0204_R008_T31SCA_20170413T104'
'021', safe_format=True, data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase('L1C_02.05',
AwsProductRequest('S2A_MSIL1C_20171012T112111_N0205_R037_T29SQC_20171012T112'
'713', safe_format=True, data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase('L1C_02.06',
AwsProductRequest('S2A_MSIL1C_20180331T212521_N0206_R043_T07WFR_20180401T005'
'612', safe_format=True, data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase('L2A_02.01',
AwsProductRequest('S2A_USER_PRD_MSIL2A_PDMC_20160310T041843_R138_V20160308T1'
'31142_20160308T131142', safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase('L2A_02.05', # L2A_02.04 is the same
AwsProductRequest('S2A_MSIL2A_20170827T105651_N0205_R094_T31WFN_20170827T105'
'652', safe_format=True, data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase('L2A_02.06',
AwsProductRequest('S2B_MSIL2A_20180216T102059_N0206_R065_T35VLL_20180216T122'
'659', safe_format=True, data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase('L2A_02.07',
AwsProductRequest('S2A_MSIL2A_20180402T151801_N0207_R068_T33XWJ_20180402T202'
'222', safe_format=True, data_folder=cls.INPUT_FOLDER))]
sys.path.append(r'C:\Program Files\QGIS 3.12\apps\Python37\Scripts')
import gdal_merge as gm
INSTANCE_ID = 'aa356db4-d4b1-4e8a-9467-74d8a832b85a' # In case you put instance ID into configuration file you can leave this unchanged
if INSTANCE_ID:
config = SHConfig()
config.instance_id = INSTANCE_ID
else:
config = None
product_id = 'S2A_MSIL2A_20190821T085601_N0213_R007_T36UUA_20190821T115206'
data_folder = './AwsData'
product_request = AwsProductRequest(product_id=product_id,
data_folder=data_folder,
safe_format=True)
product_request.save_data()
for i in [10, 20, 60]:
path = 'A:\lab4\S2A_MSIL2A_20190821T085601_N0213_R007_T36UUA_20190821T115206.SAFE\GRANULE\L2A_T36UUA_A021740_20190821T085815\IMG_DATA\R{}m\\'.format(
i)
in1 = glob.glob(str(path) + ('*B02_{}m.jp2').format(i))
in2 = glob.glob(str(path) + '*B03_{}m.jp2'.format(i))
in3 = glob.glob(str(path) + '*B04_{}m.jp2'.format(i))
in4 = glob.glob(str(path) + '*_*_*8*_{}m.jp2'.format(i))
gm.main([
'', '-separate', '-o', 'AR{}.tif'.format(i), in1[0], in2[0], in3[0],
in4[0]
])
def setUpClass(cls):
super().setUpClass()
cls.test_cases = [
cls.SafeTestCase(
'L1C_02.01',
AwsProductRequest(
'S2A_OPER_PRD_MSIL1C_PDMC_20151218T020842_R115_V20151217T224602_'
'20151217T224602',
bands=AwsConstants.S2_L1C_BANDS,
metafiles=AwsConstants.S2_L1C_METAFILES,
tile_list=['T59HNA'],
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L1C_02.01_tile',
AwsTileRequest(tile='29KQB',
time='2016-04-12',
aws_index=None,
data_collection=DataCollection.SENTINEL2_L1C,
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L1C_02.02',
AwsProductRequest(
'S2A_OPER_PRD_MSIL1C_PDMC_20160606T232310_R121_V20160526T0'
'84351_20160526T084351.SAFE',
tile_list=['34HCF'],
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L1C_02.04_old',
AwsProductRequest(
'S2A_OPER_PRD_MSIL1C_PDMC_20160910T174323_R071_V20160701T2'
'04642_20160701T204643',
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L1C_02.04',
AwsProductRequest(
'S2A_MSIL1C_20170413T104021_N0204_R008_T31SCA_20170413T104021',
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L1C_02.05',
AwsProductRequest(
'S2A_MSIL1C_20171012T112111_N0205_R037_T29SQC_20171012T112713',
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L1C_02.06',
AwsProductRequest(
'S2A_MSIL1C_20180331T212521_N0206_R043_T07WFR_20180401T005612',
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L1C_02.06_2',
AwsProductRequest(
'S2A_MSIL1C_20181004T175151_N0206_R141_T18XVM_20190219T160358',
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L1C_02.07',
AwsProductRequest(
'S2A_MSIL1C_20181119T031011_N0207_R075_T50TLK_20181119T061056',
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L1C_02.07_2',
AwsProductRequest(
'S2A_MSIL1C_20190129T143751_N0207_R096_T20LLK_20190225T132350',
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L2A_02.01',
AwsProductRequest(
'S2A_USER_PRD_MSIL2A_PDMC_20160310T041843_R138_V20160308T131142_'
'20160308T131142',
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L2A_02.05', # L2A_02.04 is the same
AwsProductRequest(
'S2A_MSIL2A_20170827T105651_N0205_R094_T31WFN_20170827T105652',
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L2A_02.06',
AwsProductRequest(
'S2B_MSIL2A_20180216T102059_N0206_R065_T35VLL_20180216T122659',
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L2A_02.07',
AwsProductRequest(
'S2A_MSIL2A_20180402T151801_N0207_R068_T33XWJ_20180402T202222',
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L2A_02.08',
AwsProductRequest(
'S2A_MSIL2A_20181005T104021_N0208_R008_T34WEU_20181007T220806',
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L2A_02.09',
AwsProductRequest(
'S2B_MSIL2A_20181029T093109_N0209_R136_T35UMQ_20181029T122414',
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L2A_02.10',
AwsProductRequest(
'S2B_MSIL2A_20181115T110319_N0210_R094_T32VLJ_20181115T142501',
safe_format=True,
data_folder=cls.INPUT_FOLDER)),
cls.SafeTestCase(
'L2A_02.11',
AwsProductRequest(
'S2B_MSIL2A_20190310T235739_N0211_R030_T57MVM_20190311T013927',
safe_format=True,
data_folder=cls.INPUT_FOLDER))
]
allow_pickle=True)['arr_0'][()]
for scene in list(s1scenes.keys()):
s1list.append(s1scenes[scene]['title'])
s2scenes = np.load('query_results/%s_query_results_sentinel_2.npz' % site,
allow_pickle=True)['arr_0'][()]
for scene in list(s2scenes.keys()):
s2list.append(s2scenes[scene]['title'])
s1list = np.unique(s1list)
s2list = np.unique(s2list)
# download the sentinel 1 data
for id in s1list:
GRDrequest = AwsProductRequest(product_id=id,
data_folder='../DATA/Sentinel1/GRD/',
config=config,
safe_format=True)
GRDrequest.save_data() # save's data to disk (this stage works)
# download the sentinel 2 data
for id in s2list:
L2Arequest = AwsProductRequest(product_id=id,
data_folder='../DATA/Sentinel2/L2A/',
config=config,
safe_format=True)
L2Arequest.save_data()
# get tile and date information to retrieve corresponing L1C tile
tile = id.split('_')[5]
year = id.split('_')[6][:4]
month = id.split('_')[6][4:6]
from sentinelhub import WebFeatureService, BBox, CRS, DataSource
from sentinelhub import AwsTileRequest, WmsRequest, AwsProductRequest
import config_file, time
INSTANCE_ID = '97b3c965-e57f-4a06-9ddc-e1ec478d930a' # In case you put instance ID into cofniguration file you can leave this unchanged
search_bbox = BBox(bbox=[46.16, -16.15, 46.51, -15.58], crs=CRS.WGS84)
search_time_interval = ('2017-12-01T00:00:00', '2017-12-15T23:59:59')
wfs_iterator = WebFeatureService(
search_bbox,
search_time_interval,
data_source=DataSource.LANDSAT8,
instance_id='f438079a-b072-4285-b154-037b0627d223')
request = AwsProductRequest(product_id=x[0]['properties']['id'],
data_folder='')
tt = time.time()
metafiles = []
for (tile_name, t_time, aws_index) in wfs_iterator.get_tiles():
tile_request = AwsTileRequest(
tile=tile_name,
time=t_time,
aws_index=aws_index,
data_folder=data_folder) #metafiles=metafiles
tile_request.save_data()
print(tt - time.time())
request = AwsProductRequest(product_id=product_id, data_folder=data_folder)
tile_id = 'S2A_OPER_MSI_L1C_TL_MTI__20151219T100121_A002563_T38TML_N02.01'
tile_name, time, aws_index = sentinelhub.AwsTile.tile_id_to_tile(tile_id)