def test_make_absolute_href_windows(self):
utils._pathlib = ntpath
try:
# Test cases of (source_href, start_href, expected)
test_cases = [
('item.json', 'C:\\a\\b\\c\\catalog.json',
'c:\\a\\b\\c\\item.json'),
('.\\item.json', 'C:\\a\\b\\c\\catalog.json',
'c:\\a\\b\\c\\item.json'),
('.\\z\\item.json', 'Z:\\a\\b\\c\\catalog.json',
'z:\\a\\b\\c\\z\\item.json'),
('..\\item.json', 'a:\\a\\b\\c\\catalog.json',
'a:\\a\\b\\item.json'),
('item.json', 'HTTPS://stacspec.org/a/b/c/catalog.json',
'https://stacspec.org/a/b/c/item.json'),
('./item.json', 'https://stacspec.org/a/b/c/catalog.json',
'https://stacspec.org/a/b/c/item.json'),
('./z/item.json', 'https://stacspec.org/a/b/c/catalog.json',
'https://stacspec.org/a/b/c/z/item.json'),
('../item.json', 'https://stacspec.org/a/b/c/catalog.json',
'https://stacspec.org/a/b/item.json')
]
for source_href, start_href, expected in test_cases:
actual = make_absolute_href(source_href, start_href)
self.assertEqual(actual, expected)
finally:
utils._pathlib = os.path
def test_copy_to_relative(self):
cat = TestCases.planet_disaster()
with TemporaryDirectory() as tmp_dir:
cat.make_all_asset_hrefs_absolute()
cat.normalize_hrefs(tmp_dir)
cat.save(catalog_type=pystac.CatalogType.ABSOLUTE_PUBLISHED)
cat2_dir = os.path.join(tmp_dir, 'second')
command = [
'copy', '-t', 'SELF_CONTAINED', '-a',
cat.get_self_href(), cat2_dir
]
self.run_command(command)
cat2 = pystac.read_file(os.path.join(cat2_dir, 'collection.json'))
for item in cat2.get_all_items():
item_href = item.get_self_href()
for asset in item.assets.values():
href = asset.href
self.assertFalse(is_absolute_href(href))
common_path = os.path.commonpath([
os.path.dirname(item_href),
make_absolute_href(href, item_href)
])
self.assertTrue(common_path, os.path.dirname(item_href))
def launch_browser(catalog_uri):
catalog_uri = make_absolute_href(catalog_uri)
catalog_dir = os.path.dirname(catalog_uri)
catalog_filename = os.path.basename(catalog_uri)
with TemporaryDirectory() as tmp_dir:
for fname in ['Dockerfile-node', 'Dockerfile-titiler']:
p = os.path.join(os.path.dirname(__file__), fname)
shutil.copy(p, os.path.join(tmp_dir, fname))
# Load template docker-compose
template_loader = jinja2.FileSystemLoader(
searchpath=os.path.dirname(__file__))
template_env = jinja2.Environment(loader=template_loader)
template_file = "docker-compose.yml.template"
template = template_env.get_template(template_file)
rendered_docker_compose = template.render(
catalog_dir=catalog_dir, catalog_filename=catalog_filename)
with open(os.path.join(tmp_dir, "docker-compose.yml"), 'w') as f:
f.write(rendered_docker_compose)
curdir = os.path.abspath(os.curdir)
try:
os.chdir(tmp_dir)
p = Popen(['docker-compose', 'up'])
p.wait()
finally:
call(['docker-compose', 'kill'])
p.terminate()
os.chdir(curdir)
def test_make_absolute_href(self) -> None:
# Test cases of (source_href, start_href, expected)
test_cases = [
("item.json", "/a/b/c/catalog.json", "/a/b/c/item.json"),
("./item.json", "/a/b/c/catalog.json", "/a/b/c/item.json"),
("./z/item.json", "/a/b/c/catalog.json", "/a/b/c/z/item.json"),
("../item.json", "/a/b/c/catalog.json", "/a/b/item.json"),
(
"item.json",
"https://stacspec.org/a/b/c/catalog.json",
"https://stacspec.org/a/b/c/item.json",
),
(
"./item.json",
"https://stacspec.org/a/b/c/catalog.json",
"https://stacspec.org/a/b/c/item.json",
),
(
"./z/item.json",
"https://stacspec.org/a/b/c/catalog.json",
"https://stacspec.org/a/b/c/z/item.json",
),
(
"../item.json",
"https://stacspec.org/a/b/c/catalog.json",
"https://stacspec.org/a/b/item.json",
),
("http://localhost:8000", None, "http://localhost:8000"),
]
for source_href, start_href, expected in test_cases:
actual = make_absolute_href(source_href, start_href)
self.assertEqual(actual, expected)
def set_self_href(self, href):
"""Sets the absolute HREF that is represented by the ``rel == 'self'``
:class:`~pystac.Link`.
Changing the self HREF of the item will ensure that all asset HREFs
remain valid. If asset HREFs are relative, the HREFs will change
to point to the same location based on the new item self HREF,
either by making them relative to the new location or making them
absolute links if the new location does not share the same protocol
as the old location.
Args:
href (str): The absolute HREF of this object. If the given HREF
is not absolute, it will be transformed to an absolute
HREF based on the current working directory. If this is None
the call will clear the self HREF link.
"""
prev_href = self.get_self_href()
super().set_self_href(href)
new_href = self.get_self_href() # May have been made absolute.
if prev_href is not None:
# Make sure relative asset links remain valid.
for asset in self.assets.values():
asset_href = asset.href
if not is_absolute_href(asset_href):
abs_href = make_absolute_href(asset_href, prev_href)
new_relative_href = make_relative_href(abs_href, new_href)
asset.href = new_relative_href
def from_file(cls, href):
"""Reads a STACObject implementation from a file.
Args:
href (str): The HREF to read the object from.
Returns:
The specific STACObject implementation class that is represented
by the JSON read from the file located at HREF.
"""
if not is_absolute_href(href):
href = make_absolute_href(href)
d = STAC_IO.read_json(href)
if cls == STACObject:
o = STAC_IO.stac_object_from_dict(d, href=href)
else:
o = cls.from_dict(d, href=href)
# Set the self HREF, if it's not already set to something else.
if o.get_self_href() is None:
o.set_self_href(href)
# If this is a root catalog, set the root to the catalog instance.
root_link = o.get_root_link()
if root_link is not None:
if not root_link.is_resolved():
if root_link.get_absolute_href() == href:
o.set_root(o, link_type=root_link.link_type)
return o
def normalize_hrefs(self, root_href):
if not is_absolute_href(root_href):
root_href = make_absolute_href(root_href, os.getcwd(), start_is_dir=True)
old_self_href = self.get_self_href()
new_self_href = os.path.join(root_href, '{}.json'.format(self.id))
self.set_self_href(new_self_href)
# Make sure relative asset links remain valid.
# This will only work if there is a self href set.
for asset in self.assets.values():
asset_href = asset.href
if not is_absolute_href(asset_href):
if old_self_href is not None:
abs_href = make_absolute_href(asset_href, old_self_href)
new_relative_href = make_relative_href(abs_href, new_self_href)
asset.href = new_relative_href
def __init__(self,
item_metadata,
item_assets,
base_dir,
metadata_href=None):
self.item_metadata = item_metadata
self.item_assets = item_assets
self.base_dir = make_absolute_href(base_dir)
self.metadata_href = metadata_href
def validate_catalog_link_type(href, link_type, should_include_self):
cat_dict = STAC_IO.read_json(href)
cat = STACObject.from_file(href)
for link in cat.get_links():
if not link.rel == 'self':
self.assertEqual(link.link_type, link_type)
rels = set([link['rel'] for link in cat_dict['links']])
self.assertEqual('self' in rels, should_include_self)
for child_link in cat.get_child_links():
child_href = make_absolute_href(child_link.target, href)
validate_catalog_link_type(child_href, link_type,
catalog_type == CatalogType.ABSOLUTE_PUBLISHED)
for item_link in cat.get_item_links():
item_href = make_absolute_href(item_link.target, href)
validate_item_link_type(item_href, link_type,
catalog_type == CatalogType.ABSOLUTE_PUBLISHED)
def merge_common_properties(item_dict, collection_cache=None, json_href=None):
"""Merges Collection properties into an Item.
Args:
item_dict: JSON dict of the Item which properties should be merged
into.
collection_cache: Optional cache of Collection JSON that has previously
read. Keyed to either the Collection ID or an HREF.
json_href: The HREF of the file that this JSON comes from. Used
to resolve relative paths.
Returns:
bool: True if Collection properties have been merged, otherwise False.
"""
properties_merged = False
collection = None
collection_href = None
# Try the cache if we have a collection ID.
if 'collection' in item_dict:
collection_id = item_dict['collection']
if collection_cache is not None:
collection = collection_cache.get(collection_id)
# Next, try the collection link.
if collection is None:
links = item_dict['links']
collection_link = next((l for l in links if l['rel'] == 'collection'),
None)
if collection_link is not None:
collection_href = collection_link['href']
if json_href is not None:
collection_href = make_absolute_href(collection_href,
json_href)
if collection_cache is not None:
collection = collection_cache.get(collection_href)
if collection is None:
collection = STAC_IO.read_json(collection_href)
if collection is not None:
if 'properties' in collection:
for k in collection['properties']:
if k not in item_dict['properties']:
properties_merged = True
item_dict['properties'][k] = collection['properties'][k]
if collection_cache is not None and collection[
'id'] not in collection_cache:
collection_id = collection['id']
collection_cache[collection_id] = collection
if collection_href is not None:
collection_cache[collection_href] = collection
return properties_merged
def copy_command(src: str, dst: str, catalog_type: pystac.CatalogType,
copy_assets: bool,
publish_location: Optional[str]) -> None:
"""Copy a STAC Catalog or Collection at SRC to the directory
at DST.
Note: Copying a catalog will upgrade it to the latest version of STAC."""
source_catalog = pystac.read_file(make_absolute_href(src))
if not isinstance(source_catalog, pystac.Catalog):
raise click.BadArgumentUsage(f"{src} is not a STAC Catalog")
copy_catalog(source_catalog, dst, catalog_type, copy_assets,
publish_location)
def copy_catalog(source_catalog,
dest_directory,
catalog_type=None,
copy_assets=False):
catalog = source_catalog.full_copy()
dest_directory = make_absolute_href(dest_directory)
catalog.normalize_hrefs(dest_directory)
if copy_assets:
catalog = move_all_assets(catalog, copy=True, make_hrefs_relative=True)
catalog.save(catalog_type)
def validate_catalog_link_type(
href: str, link_type: str, should_include_self: bool
) -> None:
cat_dict = pystac.StacIO.default().read_json(href)
cat = pystac.read_file(href)
assert isinstance(cat, pystac.Catalog)
rels = set([link["rel"] for link in cat_dict["links"]])
self.assertEqual("self" in rels, should_include_self)
for child_link in cat.get_child_links():
child_href = make_absolute_href(child_link.href, href)
validate_catalog_link_type(
child_href,
link_type,
catalog_type == CatalogType.ABSOLUTE_PUBLISHED,
)
for item_link in cat.get_item_links():
item_href = make_absolute_href(item_link.href, href)
validate_item_link_type(
item_href, link_type, catalog_type == CatalogType.ABSOLUTE_PUBLISHED
)
def validate_all(stac_dict: Dict[str, Any],
href: str,
stac_io: Optional[pystac.StacIO] = None) -> None:
"""Validate STAC JSON and all contained catalogs, collections and items.
If this stac_dict represents a catalog or collection, this method will
recursively be called for each child link and all contained items.
Args:
stac_dict : Dictionary that is the STAC json of the object.
href : HREF of the STAC object being validated. Used for error
reporting and resolving relative links.
stac_io: Optional StacIO instance to use for reading hrefs. If None,
the StacIO.default() instance is used.
Raises:
STACValidationError: This will raise a STACValidationError if this or any
contained catalog, collection or item has a validation error.
"""
if stac_io is None:
stac_io = pystac.StacIO.default()
info = identify_stac_object(stac_dict)
# Validate this object
validate_dict(
stac_dict,
stac_object_type=info.object_type,
stac_version=str(info.version_range.latest_valid_version()),
extensions=list(info.extensions),
href=href,
)
if info.object_type != pystac.STACObjectType.ITEM:
if "links" in stac_dict:
# Account for 0.6 links
if isinstance(stac_dict["links"], dict):
links: List[Dict[str, Any]] = list(stac_dict["links"].values())
else:
links = cast(List[Dict[str, Any]], stac_dict.get("links"))
for link in links:
rel = link.get("rel")
if rel in [pystac.RelType.ITEM, pystac.RelType.CHILD]:
link_href = make_absolute_href(cast(str, link.get("href")),
start_href=href)
if link_href is not None:
d = stac_io.read_json(link_href)
validate_all(d, link_href)
def test_add_raster_to_item(self):
with TemporaryDirectory() as tmp_dir:
catalog = create_temp_catalog_copy(tmp_dir)
items = list(catalog.get_all_items())
item_path = make_absolute_href(items[0].get_self_href(),
catalog.get_self_href())
cmd = ["addraster", item_path]
self.run_command(cmd)
updated = pystac.read_file(catalog.get_self_href())
item = list(updated.get_all_items())[0]
asset = item.get_assets().get("analytic")
assert asset is not None
expected = expected_json("rasterbands.json")
assert expected == asset.to_dict().get("raster:bands")
def get_absolute_href(self):
"""Gets the absolute href for this asset, if possible.
If this Asset has no associated Item, this will return whatever the
href is (as it cannot determine the absolute path, if the asset
href is relative).
Returns:
str: The absolute HREF of this asset, or a relative HREF is an abslolute HREF
cannot be determined.
"""
if not is_absolute_href(self.href):
if self.owner is not None:
return make_absolute_href(self.href, self.owner.get_self_href())
return self.href
def get_absolute_href(self):
"""Gets the absolute href for this link, if possible.
Returns:
str: Returns this link's HREF. It attempts to derive an absolute HREF
from this link; however, if the link is relative, has no owner,
and has an unresolved target, this will return a relative HREF.
"""
if self.is_resolved():
href = self.target.get_self_href()
else:
href = self.target
if self.owner is not None:
href = make_absolute_href(href, self.owner.get_self_href())
return href
def copy_catalog(source_catalog: Catalog,
dest_directory: str,
catalog_type: Optional[CatalogType] = None,
copy_assets: bool = False,
publish_location: Optional[str] = None) -> None:
catalog = source_catalog.full_copy()
dest_directory = make_absolute_href(dest_directory)
if copy_assets:
catalog = move_all_assets(catalog, copy=True, make_hrefs_relative=True)
if publish_location is not None:
catalog.normalize_hrefs(publish_location)
catalog.save(catalog_type, dest_directory)
else:
catalog.normalize_hrefs(dest_directory)
catalog.save(catalog_type)
def get_absolute_href(self) -> Optional[str]:
"""Gets the absolute href for this asset, if possible.
If this Asset has no associated Item, and the asset HREF is a relative path,
this method will return None.
Returns:
str: The absolute HREF of this asset, or None if an absolute HREF could not
be determined.
"""
if utils.is_absolute_href(self.href):
return self.href
else:
if self.owner is not None:
return utils.make_absolute_href(self.href,
self.owner.get_self_href())
else:
return None
def from_file(cls,
href: str,
stac_io: Optional[pystac.StacIO] = None) -> "ItemCollection":
"""Reads a :class:`ItemCollection` from a JSON file.
Arguments:
href : Path to the file.
stac_io : A :class:`~pystac.StacIO` instance to use for file I/O
"""
if stac_io is None:
stac_io = pystac.StacIO.default()
if not is_absolute_href(href):
href = make_absolute_href(href)
d = stac_io.read_json(href)
return cls.from_dict(d, preserve_dict=False)
def test_make_absolute_href(self):
# Test cases of (source_href, start_href, expected)
test_cases = [('item.json', '/a/b/c/catalog.json', '/a/b/c/item.json'),
('./item.json', '/a/b/c/catalog.json', '/a/b/c/item.json'),
('./z/item.json', '/a/b/c/catalog.json', '/a/b/c/z/item.json'),
('../item.json', '/a/b/c/catalog.json', '/a/b/item.json'),
('item.json', 'https://stacspec.org/a/b/c/catalog.json',
'https://stacspec.org/a/b/c/item.json'),
('./item.json', 'https://stacspec.org/a/b/c/catalog.json',
'https://stacspec.org/a/b/c/item.json'),
('./z/item.json', 'https://stacspec.org/a/b/c/catalog.json',
'https://stacspec.org/a/b/c/z/item.json'),
('../item.json', 'https://stacspec.org/a/b/c/catalog.json',
'https://stacspec.org/a/b/item.json')]
for source_href, start_href, expected in test_cases:
actual = make_absolute_href(source_href, start_href)
self.assertEqual(actual, expected)
def resolve_stac_object(self, root=None):
"""Resolves a STAC object from the HREF of this link, if the link is not
already resolved.
Args:
root (Catalog or Collection): Optional root of the catalog for this link.
If provided, the root's resolved object cache is used to search for
previously resolved instances of the STAC object.
"""
if isinstance(self.target, str):
target_href = self.target
# If it's a relative link, base it off the parent.
if not is_absolute_href(target_href):
if self.owner is None:
raise STACError('Relative path {} encountered '
'without owner or start_href.'.format(target_href))
start_href = self.owner.get_self_href()
if start_href is None:
raise STACError('Relative path {} encountered '
'without owner "self" link set.'.format(target_href))
target_href = make_absolute_href(target_href, start_href)
obj = None
if root is not None:
obj = root._resolved_objects.get_by_href(target_href)
if obj is None:
obj = STAC_IO.read_stac_object(target_href, root=root)
obj.set_self_href(target_href)
if root is not None:
obj = root._resolved_objects.get_or_cache(obj)
obj.set_root(root, link_type=self.link_type)
else:
obj = self.target
self.target = obj
if self.owner and self.rel in ['child', 'item']:
self.target.set_parent(self.owner, link_type=self.link_type)
return self
def test_make_absolute_href_windows(self) -> None:
utils._pathlib = ntpath
try:
# Test cases of (source_href, start_href, expected)
test_cases = [
("item.json", "C:\\a\\b\\c\\catalog.json",
"C:\\a\\b\\c\\item.json"),
(".\\item.json", "C:\\a\\b\\c\\catalog.json",
"C:\\a\\b\\c\\item.json"),
(
".\\z\\item.json",
"Z:\\a\\b\\c\\catalog.json",
"Z:\\a\\b\\c\\z\\item.json",
),
("..\\item.json", "a:\\a\\b\\c\\catalog.json",
"a:\\a\\b\\item.json"),
(
"item.json",
"HTTPS://stacspec.org/a/b/c/catalog.json",
"https://stacspec.org/a/b/c/item.json",
),
(
"./item.json",
"https://stacspec.org/a/b/c/catalog.json",
"https://stacspec.org/a/b/c/item.json",
),
(
"./z/item.json",
"https://stacspec.org/a/b/c/catalog.json",
"https://stacspec.org/a/b/c/z/item.json",
),
(
"../item.json",
"https://stacspec.org/a/b/c/catalog.json",
"https://stacspec.org/a/b/item.json",
),
]
for source_href, start_href, expected in test_cases:
actual = make_absolute_href(source_href, start_href)
self.assertEqual(actual, expected)
finally:
utils._pathlib = os.path
def create_cog_asset(key, asset, path):
asset_filename, extension = os.path.splitext(os.path.split(asset.href)[1])
cog_filename = f'{asset_filename}-COG.tif'
cog_path = os.path.join(path, cog_filename)
with TemporaryDirectory() as tmp_dir:
reprojected_path = os.path.join(tmp_dir, f'reprojected{extension}')
print(f'reprojecting {asset.href}')
reproject(asset.href, reprojected_path)
print(f'cogifying {asset.href}')
cogify(reprojected_path, cog_path)
shutil.rmtree(tmp_dir, ignore_errors=True)
asset = pystac.Asset(href=make_absolute_href(cog_path),
media_type=pystac.MediaType.COG,
roles=['data'],
title=f'{asset.title} (COG)',
properties=asset.properties)
return (f'{key}-cog', asset)
def add_raster_to_item(item: Item) -> Item:
"""Adds raster extension values to an item.
Args:
item (Item): The PySTAC Item to extend.
Returns:
Item: Returns an updated Item.
This operation mutates the Item.
"""
RasterExtension.add_to(item)
for asset in item.assets.values():
if asset.roles and "data" in asset.roles:
raster = RasterExtension.ext(asset)
href = make_absolute_href(asset.href, item.get_self_href())
bands = _read_bands(href)
if bands:
raster.apply(bands)
return item
def normalize_hrefs(self, root_href):
# Normalizing requires an absolute path
if not is_absolute_href(root_href):
root_href = make_absolute_href(root_href, os.getcwd(), start_is_dir=True)
# Fully resolve the STAC to avoid linking issues.
# This particularly can happen with unresolved links that have
# relative paths.
self.fully_resolve()
for child in self.get_children():
child_root = os.path.join(root_href, '{}/'.format(child.id))
child.normalize_hrefs(child_root)
for item in self.get_items():
item_root = os.path.join(root_href, '{}'.format(item.id))
item.normalize_hrefs(item_root)
self.set_self_href(os.path.join(root_href, self.DEFAULT_FILE_NAME))
return self
def make_asset_hrefs_absolute(self):
"""Modify each asset's HREF to be absolute.
Any asset HREFs that are relative will be modified to absolute based on this
item's self HREF.
Returns:
Item: self
"""
self_href = None
for asset in self.assets.values():
href = asset.href
if not is_absolute_href(href):
if self_href is None:
self_href = self.get_self_href()
if self_href is None:
raise STACError('Cannot make relative asset HREFs absolute '
'if no self_href is set.')
asset.href = make_absolute_href(asset.href, self_href)
return self
def validate_all(stac_dict, href):
"""Validate STAC JSON and all contained catalogs, collections and items.
If this stac_dict represents a catalog or collection, this method will
recursively be called for each child link and all contained items.
Args:
stac_dict (dict): Dictionary that is the STAC json of the object.
href (str): HREF of the STAC object being validated. Used for error
reporting and resolving relative links.
Raises:
STACValidationError: This will raise a STACValidationError if this or any contained
catalog, collection or item has a validation error.
"""
info = identify_stac_object(stac_dict)
# Validate this object
validate_dict(stac_dict,
stac_object_type=info.object_type,
stac_version=info.version_range.latest_valid_version(),
extensions=info.common_extensions,
href=href)
if info.object_type != pystac.STACObjectType.ITEM:
links = stac_dict.get('links')
if links is not None:
# Account for 0.6 links
if isinstance(links, dict):
links = list(links.values())
for link in links:
rel = link.get('rel')
if rel in ['item', 'child']:
link_href = make_absolute_href(link.get('href'),
start_href=href)
if link_href is not None:
d = pystac.STAC_IO.read_json(link_href)
validate_all(d, link_href)
def __init__(
self,
rel: Union[str, pystac.RelType],
target: Union[str, "STACObject_Type"],
media_type: Optional[str] = None,
title: Optional[str] = None,
extra_fields: Optional[Dict[str, Any]] = None,
) -> None:
self.rel = rel
if isinstance(target, str):
if rel == pystac.RelType.SELF:
self._target_href = make_absolute_href(target)
else:
self._target_href = target
self._target_object = None
else:
self._target_href = None
self._target_object = target
self.media_type = media_type
self.title = title
self.extra_fields = extra_fields or {}
self.owner = None
def add_raster_to_item(item: Item) -> Item:
"""Adds raster extension values to an item.
Args:
item (Item): The PySTAC Item to extend.
Returns:
Item: Returns an updated Item.
This operation mutates the Item.
"""
RasterExtension.add_to(item)
for asset in item.assets.values():
if asset.roles and "data" in asset.roles:
raster = RasterExtension.ext(asset)
bands = []
href = make_absolute_href(asset.href, item.get_self_href())
dataset = gdal.Open(href, gdal.GA_ReadOnly)
for nband in range(dataset.RasterCount):
gdal_band = dataset.GetRasterBand(nband + 1)
band = RasterBand.create()
band.nodata = gdal_band.GetNoDataValue()
band.spatial_resolution = dataset.GetGeoTransform()[1]
band.data_type = DataType(
gdal.GetDataTypeName(gdal_band.DataType).lower())
minimum = gdal_band.GetMinimum()
maximum = gdal_band.GetMaximum()
if not minimum or not max:
minimum, maximum = gdal_band.ComputeRasterMinMax(True)
band.statistics = Statistics.create(minimum=minimum,
maximum=maximum)
hist_data = gdal_band.GetHistogram(minimum, maximum,
NUM_BUCKETS)
band.histogram = Histogram.create(NUM_BUCKETS, minimum,
maximum, hist_data)
bands.append(band)
if bands:
raster.apply(bands)
return item
