def test_parses(self):
ang_path = TestData.get_path(
'data-files/landsat/assets/LC08_L2SP_005009_20150710_20200908_02_T2_ANG.txt'
)
ang_path = TestData.get_path(
'data-files/landsat/assets3/LC08_L2SP_008059_20191201_20200825_02_T1_ANG.txt'
)
ang_metadata = AngMetadata.from_file(ang_path)
# self.assertEqual(ang_metadata.scene_id, "LC80050092015191LGN01")
# Check that the proj_bbox from the MTL lines up with
# the derived geometry.
mtl_metadata = MtlMetadata.from_file(
ang_path.replace('_ANG.txt', '_MTL.xml'))
ang_geom = shape(ang_metadata.get_scene_geometry(mtl_metadata.bbox))
proj_bbox = mtl_metadata.proj_bbox
proj_bbox_shp = box(*proj_bbox)
reproj_bbox_shp = shape(
reproject_geom(f"epsg:{mtl_metadata.epsg}", "epsg:4326",
mapping(proj_bbox_shp)))
ang_geom_shp = shape(ang_geom)
self.assertLess((ang_geom_shp - reproj_bbox_shp).area,
0.0001 * ang_geom_shp.area)
def geometry(self) -> dict:
"""Returns this item's geometry in WGS84."""
original_bbox = [
float(self.minx),
float(self.miny),
float(self.maxx),
float(self.maxy)
]
return reproject_geom(THREEDEP_CRS, "EPSG:4326",
mapping(box(*original_bbox)))
def check_proj_bbox(item):
bbox = item.bbox
bbox_shp = box(*bbox)
proj_bbox = item.ext.projection.bbox
proj_bbox_shp = box(*proj_bbox)
reproj_bbox_shp = shape(
reproject_geom(f"epsg:{item.ext.projection.epsg}", "epsg:4326",
mapping(proj_bbox_shp)))
self.assertLess((reproj_bbox_shp - bbox_shp).area,
0.0001 * reproj_bbox_shp.area)
def create_cogs(hdf_path: str, xml_metadata: XmlMetadata,
output_path: str) -> Dict[str, str]:
"""Create COGs from an HDF asset and an XmlMetadata
Args:
hdf_path: Path to the ASTER L1T 003 HDF EOS data
xml_metadata: The XmlMetadata representing this ASTER scene.
output_path: The directory to which the cogs will be written.
"""
logger.info(f'Creating COGs and writing to {output_path}...')
file_name = os.path.basename(hdf_path)
aster_id = AsterSceneId.from_path(file_name)
with rio.open(hdf_path) as ds:
subdatasets = ds.subdatasets
# Gather the subdatasets by sensor, sorted by band number
sensor_to_subdatasets = defaultdict(list)
for subdataset in subdatasets:
m = re.search(r':?([\w]+)_Swath:ImageData([\d]+)', subdataset)
if m is None:
raise ValueError(
'Unexpected subdataset {} - is this a non-standard ASTER L1T 003 HDF-EOS file?'
.format(subdataset))
sensor = m.group(1)
band_order = m.group(2)
sensor_to_subdatasets[sensor].append((subdataset, band_order))
# Sort by band_order
for k in sensor_to_subdatasets:
sensor_to_subdatasets[k] = [
x for x in sorted(sensor_to_subdatasets[k], key=lambda x: x[1])
]
geom, _ = xml_metadata.geometries
crs = 'epsg:{}'.format(xml_metadata.epsg)
reprojected_geom = reproject_geom('epsg:4326', crs, geom)
bounds = list(shape(reprojected_geom).bounds)
result = {}
with TemporaryDirectory() as tmp_dir:
for sensor, subdataset_info in sensor_to_subdatasets.items():
result[sensor] = _create_cog_for_sensor(
sensor,
aster_id.file_prefix,
tmp_dir=tmp_dir,
output_dir=output_path,
bounds=bounds,
crs=crs,
subdataset_info=subdataset_info)
return result
def check_proj_bbox(item):
pb = mapping(
box(*item.ext.projection.get_bbox(item.assets['visual-10m'])))
proj_geom = shape(
reproject_geom(f'epsg:{item.ext.projection.epsg}', 'epsg:4326',
pb))
item_geom = shape(item.geometry)
difference_area = item_geom.difference(proj_geom).area
raster_area = proj_geom.area
# We expect the footprint to be in the raster
# bounds, so any difference should be relatively very low
# and due to reprojection.
self.assertLess(difference_area / raster_area, 0.005)
def test_parses_xml_utm(self):
mtl_path = TestData.get_path(
'data-files/landsat/assets/LC08_L2SP_005009_20150710_20200908_02_T2_MTL.xml'
)
mtl_metadata = MtlMetadata.from_file(mtl_path)
# Datetime
dt = mtl_metadata.scene_datetime
self.assertEqual(dt.month, 7)
self.assertEqual(dt.minute, 34)
# epsg
epsg = mtl_metadata.epsg
self.assertEqual(epsg, 32624)
# bboxes
bbox = mtl_metadata.bbox
bbox_shp = box(*bbox)
proj_bbox = mtl_metadata.proj_bbox
proj_bbox_shp = box(*proj_bbox)
reproj_bbox_shp = shape(
reproject_geom(f"epsg:{epsg}", "epsg:4326",
mapping(proj_bbox_shp)))
self.assertLess((reproj_bbox_shp - bbox_shp).area,
0.0001 * reproj_bbox_shp.area)
# Cloud Cover
cloud_cover = mtl_metadata.cloud_cover
self.assertEqual(cloud_cover, 54.65)
# View
off_nadir = mtl_metadata.off_nadir
self.assertEqual(off_nadir, 0.0)
sun_azimuth = mtl_metadata.sun_azimuth
self.assertEqual(sun_azimuth, 177.8846007)
sun_elevation = mtl_metadata.sun_elevation
self.assertEqual(sun_elevation, 40.0015903)
def test_parses_xml_ps(self):
mtl_path = TestData.get_path(
'data-files/landsat/assets2/LC08_L2SR_099120_20191129_20201016_02_T2_MTL.xml'
)
mtl_metadata = MtlMetadata.from_file(mtl_path)
# epsg
epsg = mtl_metadata.epsg
self.assertEqual(epsg, 3031)
# bboxes
bbox = mtl_metadata.bbox
bbox_shp = box(*bbox)
proj_bbox = mtl_metadata.proj_bbox
proj_bbox_shp = box(*proj_bbox)
reproj_bbox_shp = shape(
reproject_geom(f"epsg:{epsg}", "epsg:4326",
mapping(proj_bbox_shp)))
self.assertLess((reproj_bbox_shp - bbox_shp).area,
0.0001 * reproj_bbox_shp.area)
def create_item(state,
year,
cog_href,
fgdc_metadata_href: Optional[str],
thumbnail_href=None,
additional_providers=None):
"""Creates a STAC Item from NAIP data.
Args:
state (str): The 2-letter state code for the state this item belongs to.
year (str): The NAIP year.
fgdc_metadata_href (str): The href to the FGDC metadata
for this NAIP scene. Optional, a some NAIP scenes to not have this
(e.g. 2010)
cog_href (str): The href to the image as a COG. This needs
to be an HREF that rasterio is able to open.
thumbnail_href (str): Optional href for a thumbnail for this scene.
additional_providers(List[pystac.Provider]): Optional list of additional
providers to the USDA that will be included on this Item.
This function will read the metadata file for information to place in
the STAC item.
Returns:
pystac.Item: A STAC Item representing this NAIP scene.
"""
with rio.open(cog_href) as ds:
gsd = ds.res[0]
epsg = int(ds.crs.to_authority()[1])
image_shape = list(ds.shape)
original_bbox = list(ds.bounds)
transform = list(ds.transform)
geom = reproject_geom(ds.crs,
'epsg:4326',
mapping(box(*ds.bounds)),
precision=6)
if fgdc_metadata_href is not None:
fgdc_metadata_text = pystac.STAC_IO.read_text(fgdc_metadata_href)
fgdc = parse_fgdc_metadata(fgdc_metadata_text)
else:
fgdc = {}
if 'Distribution_Information' in fgdc:
resource_desc = fgdc['Distribution_Information'][
'Resource_Description']
else:
resource_desc = os.path.basename(cog_href)
item_id = naip_item_id(state, resource_desc)
bounds = list(shape(geom).bounds)
if any(fgdc):
dt = str_to_datetime(
fgdc['Identification_Information']['Time_Period_of_Content']
['Time_Period_Information']['Single_Date/Time']['Calendar_Date'])
else:
fname = os.path.splitext(os.path.basename(cog_href))[0]
fname_date = fname.split('_')[5]
dt = dateutil.parser.isoparse(fname_date)
properties = {'naip:state': state, 'naip:year': year}
item = pystac.Item(id=item_id,
geometry=geom,
bbox=bounds,
datetime=dt,
properties=properties)
# Common metadata
item.common_metadata.providers = [constants.USDA_PROVIDER]
if additional_providers is not None:
item.common_metadata.providers.extend(additional_providers)
item.common_metadata.gsd = gsd
# eo, for asset bands
item.ext.enable('eo')
# proj
item.ext.enable('projection')
item.ext.projection.epsg = epsg
item.ext.projection.shape = image_shape
item.ext.projection.bbox = original_bbox
item.ext.projection.transform = transform
# COG
item.add_asset(
'image',
pystac.Asset(href=cog_href,
media_type=pystac.MediaType.COG,
roles=['data'],
title="RGBIR COG tile"))
# Metadata
if any(fgdc):
item.add_asset(
'metadata',
pystac.Asset(href=fgdc_metadata_href,
media_type=pystac.MediaType.TEXT,
roles=['metadata'],
title='FGDC Metdata'))
if thumbnail_href is not None:
media_type = pystac.MediaType.JPEG
if thumbnail_href.lower().endswith('png'):
media_type = pystac.MediaType.PNG
item.add_asset(
'thumbnail',
pystac.Asset(href=thumbnail_href,
media_type=media_type,
roles=['thumbnail'],
title='Thumbnail'))
item.ext.eo.set_bands(constants.NAIP_BANDS, item.assets['image'])
return item
def test_create_cogs_then_items(self):
"""Test cogs and items so we don't have to save additional
test data"""
hdf_path = TestData.get_external_data(HDF_PATH_EXTERNAL)
xml_path = TestData.get_path(XML_PATH)
with TemporaryDirectory() as tmp_dir:
cog_cmd = [
'aster', 'create-cogs', "--hdf", hdf_path, "--xml", xml_path,
"--output", tmp_dir
]
self.run_command(cog_cmd)
cogs = [p for p in os.listdir(tmp_dir) if p.endswith('.tif')]
self.assertEqual(
set(cogs),
set([
'AST_L1T_00305032000040446_20150409135350_78838-VNIR.tif',
'AST_L1T_00305032000040446_20150409135350_78838-SWIR.tif',
'AST_L1T_00305032000040446_20150409135350_78838-TIR.tif'
]))
# Check band names, and that there is variable data
for cog in cogs:
sensor = os.path.splitext(cog)[0].split('-')[-1]
with rio.open(os.path.join(tmp_dir, cog)) as ds:
for band_name in ds.descriptions:
self.assertTrue(sensor in band_name)
self.assertTrue(ds.read().any())
vnir_cog_fname = next(c for c in cogs if 'VNIR' in c)
swir_cog_fname = next(c for c in cogs if 'SWIR' in c)
tir_cog_fname = next(c for c in cogs if 'TIR' in c)
stac_cmd = [
'aster', 'create-item', '--xml', xml_path, '--vnir',
os.path.join(tmp_dir, vnir_cog_fname), '--swir',
os.path.join(tmp_dir, swir_cog_fname), '--tir',
os.path.join(tmp_dir, tir_cog_fname), '--hdf', hdf_path,
'--vnir-browse', VNIR_BROWSE_PATH, '--tir-browse',
TIR_BROWSE_PATH, '--qa-browse', QA_BROWSE_PATH, '--qa-txt',
QA_TXT_PATH, '--output', tmp_dir
]
self.run_command(stac_cmd)
jsons = [p for p in os.listdir(tmp_dir) if p.endswith('.json')]
self.assertEqual(len(jsons), 1)
item_path = os.path.join(tmp_dir, jsons[0])
item: pystac.Item = cast(pystac.Item, pystac.read_file(item_path))
item.validate()
self.assertEqual(
set(item.assets.keys()),
set([
VNIR_SENSOR, SWIR_SENSOR, TIR_SENSOR, HDF_ASSET_KEY,
XML_ASSET_KEY, VNIR_BROWSE_ASSET_KEY, TIR_BROWSE_ASSET_KEY,
QA_BROWSE_ASSET_KEY, QA_TXT_ASSET_KEY
]))
# Check that the proj bbox and item geom align
crs = f'epsg:{item.ext.projection.epsg}'
for asset_key in [VNIR_SENSOR, SWIR_SENSOR, TIR_SENSOR]:
proj_bbox_shp = box(
*item.ext.projection.get_bbox(item.assets[asset_key]))
projected_shp = shape(
reproject_geom('epsg:4326', crs, item.geometry))
self.assertTrue(proj_bbox_shp.covers(projected_shp))