def test_make_absolute_href_windows(self): utils._pathlib = ntpath try: # Test cases of (source_href, start_href, expected) test_cases = [ ('item.json', 'C:\\a\\b\\c\\catalog.json', 'c:\\a\\b\\c\\item.json'), ('.\\item.json', 'C:\\a\\b\\c\\catalog.json', 'c:\\a\\b\\c\\item.json'), ('.\\z\\item.json', 'Z:\\a\\b\\c\\catalog.json', 'z:\\a\\b\\c\\z\\item.json'), ('..\\item.json', 'a:\\a\\b\\c\\catalog.json', 'a:\\a\\b\\item.json'), ('item.json', 'HTTPS://stacspec.org/a/b/c/catalog.json', 'https://stacspec.org/a/b/c/item.json'), ('./item.json', 'https://stacspec.org/a/b/c/catalog.json', 'https://stacspec.org/a/b/c/item.json'), ('./z/item.json', 'https://stacspec.org/a/b/c/catalog.json', 'https://stacspec.org/a/b/c/z/item.json'), ('../item.json', 'https://stacspec.org/a/b/c/catalog.json', 'https://stacspec.org/a/b/item.json') ] for source_href, start_href, expected in test_cases: actual = make_absolute_href(source_href, start_href) self.assertEqual(actual, expected) finally: utils._pathlib = os.path
def test_copy_to_relative(self): cat = TestCases.planet_disaster() with TemporaryDirectory() as tmp_dir: cat.make_all_asset_hrefs_absolute() cat.normalize_hrefs(tmp_dir) cat.save(catalog_type=pystac.CatalogType.ABSOLUTE_PUBLISHED) cat2_dir = os.path.join(tmp_dir, 'second') command = [ 'copy', '-t', 'SELF_CONTAINED', '-a', cat.get_self_href(), cat2_dir ] self.run_command(command) cat2 = pystac.read_file(os.path.join(cat2_dir, 'collection.json')) for item in cat2.get_all_items(): item_href = item.get_self_href() for asset in item.assets.values(): href = asset.href self.assertFalse(is_absolute_href(href)) common_path = os.path.commonpath([ os.path.dirname(item_href), make_absolute_href(href, item_href) ]) self.assertTrue(common_path, os.path.dirname(item_href))
def launch_browser(catalog_uri): catalog_uri = make_absolute_href(catalog_uri) catalog_dir = os.path.dirname(catalog_uri) catalog_filename = os.path.basename(catalog_uri) with TemporaryDirectory() as tmp_dir: for fname in ['Dockerfile-node', 'Dockerfile-titiler']: p = os.path.join(os.path.dirname(__file__), fname) shutil.copy(p, os.path.join(tmp_dir, fname)) # Load template docker-compose template_loader = jinja2.FileSystemLoader( searchpath=os.path.dirname(__file__)) template_env = jinja2.Environment(loader=template_loader) template_file = "docker-compose.yml.template" template = template_env.get_template(template_file) rendered_docker_compose = template.render( catalog_dir=catalog_dir, catalog_filename=catalog_filename) with open(os.path.join(tmp_dir, "docker-compose.yml"), 'w') as f: f.write(rendered_docker_compose) curdir = os.path.abspath(os.curdir) try: os.chdir(tmp_dir) p = Popen(['docker-compose', 'up']) p.wait() finally: call(['docker-compose', 'kill']) p.terminate() os.chdir(curdir)
def test_make_absolute_href(self) -> None: # Test cases of (source_href, start_href, expected) test_cases = [ ("item.json", "/a/b/c/catalog.json", "/a/b/c/item.json"), ("./item.json", "/a/b/c/catalog.json", "/a/b/c/item.json"), ("./z/item.json", "/a/b/c/catalog.json", "/a/b/c/z/item.json"), ("../item.json", "/a/b/c/catalog.json", "/a/b/item.json"), ( "item.json", "https://stacspec.org/a/b/c/catalog.json", "https://stacspec.org/a/b/c/item.json", ), ( "./item.json", "https://stacspec.org/a/b/c/catalog.json", "https://stacspec.org/a/b/c/item.json", ), ( "./z/item.json", "https://stacspec.org/a/b/c/catalog.json", "https://stacspec.org/a/b/c/z/item.json", ), ( "../item.json", "https://stacspec.org/a/b/c/catalog.json", "https://stacspec.org/a/b/item.json", ), ("http://localhost:8000", None, "http://localhost:8000"), ] for source_href, start_href, expected in test_cases: actual = make_absolute_href(source_href, start_href) self.assertEqual(actual, expected)
def set_self_href(self, href): """Sets the absolute HREF that is represented by the ``rel == 'self'`` :class:`~pystac.Link`. Changing the self HREF of the item will ensure that all asset HREFs remain valid. If asset HREFs are relative, the HREFs will change to point to the same location based on the new item self HREF, either by making them relative to the new location or making them absolute links if the new location does not share the same protocol as the old location. Args: href (str): The absolute HREF of this object. If the given HREF is not absolute, it will be transformed to an absolute HREF based on the current working directory. If this is None the call will clear the self HREF link. """ prev_href = self.get_self_href() super().set_self_href(href) new_href = self.get_self_href() # May have been made absolute. if prev_href is not None: # Make sure relative asset links remain valid. for asset in self.assets.values(): asset_href = asset.href if not is_absolute_href(asset_href): abs_href = make_absolute_href(asset_href, prev_href) new_relative_href = make_relative_href(abs_href, new_href) asset.href = new_relative_href
def from_file(cls, href): """Reads a STACObject implementation from a file. Args: href (str): The HREF to read the object from. Returns: The specific STACObject implementation class that is represented by the JSON read from the file located at HREF. """ if not is_absolute_href(href): href = make_absolute_href(href) d = STAC_IO.read_json(href) if cls == STACObject: o = STAC_IO.stac_object_from_dict(d, href=href) else: o = cls.from_dict(d, href=href) # Set the self HREF, if it's not already set to something else. if o.get_self_href() is None: o.set_self_href(href) # If this is a root catalog, set the root to the catalog instance. root_link = o.get_root_link() if root_link is not None: if not root_link.is_resolved(): if root_link.get_absolute_href() == href: o.set_root(o, link_type=root_link.link_type) return o
def normalize_hrefs(self, root_href): if not is_absolute_href(root_href): root_href = make_absolute_href(root_href, os.getcwd(), start_is_dir=True) old_self_href = self.get_self_href() new_self_href = os.path.join(root_href, '{}.json'.format(self.id)) self.set_self_href(new_self_href) # Make sure relative asset links remain valid. # This will only work if there is a self href set. for asset in self.assets.values(): asset_href = asset.href if not is_absolute_href(asset_href): if old_self_href is not None: abs_href = make_absolute_href(asset_href, old_self_href) new_relative_href = make_relative_href(abs_href, new_self_href) asset.href = new_relative_href
def __init__(self, item_metadata, item_assets, base_dir, metadata_href=None): self.item_metadata = item_metadata self.item_assets = item_assets self.base_dir = make_absolute_href(base_dir) self.metadata_href = metadata_href
def validate_catalog_link_type(href, link_type, should_include_self): cat_dict = STAC_IO.read_json(href) cat = STACObject.from_file(href) for link in cat.get_links(): if not link.rel == 'self': self.assertEqual(link.link_type, link_type) rels = set([link['rel'] for link in cat_dict['links']]) self.assertEqual('self' in rels, should_include_self) for child_link in cat.get_child_links(): child_href = make_absolute_href(child_link.target, href) validate_catalog_link_type(child_href, link_type, catalog_type == CatalogType.ABSOLUTE_PUBLISHED) for item_link in cat.get_item_links(): item_href = make_absolute_href(item_link.target, href) validate_item_link_type(item_href, link_type, catalog_type == CatalogType.ABSOLUTE_PUBLISHED)
def merge_common_properties(item_dict, collection_cache=None, json_href=None): """Merges Collection properties into an Item. Args: item_dict: JSON dict of the Item which properties should be merged into. collection_cache: Optional cache of Collection JSON that has previously read. Keyed to either the Collection ID or an HREF. json_href: The HREF of the file that this JSON comes from. Used to resolve relative paths. Returns: bool: True if Collection properties have been merged, otherwise False. """ properties_merged = False collection = None collection_href = None # Try the cache if we have a collection ID. if 'collection' in item_dict: collection_id = item_dict['collection'] if collection_cache is not None: collection = collection_cache.get(collection_id) # Next, try the collection link. if collection is None: links = item_dict['links'] collection_link = next((l for l in links if l['rel'] == 'collection'), None) if collection_link is not None: collection_href = collection_link['href'] if json_href is not None: collection_href = make_absolute_href(collection_href, json_href) if collection_cache is not None: collection = collection_cache.get(collection_href) if collection is None: collection = STAC_IO.read_json(collection_href) if collection is not None: if 'properties' in collection: for k in collection['properties']: if k not in item_dict['properties']: properties_merged = True item_dict['properties'][k] = collection['properties'][k] if collection_cache is not None and collection[ 'id'] not in collection_cache: collection_id = collection['id'] collection_cache[collection_id] = collection if collection_href is not None: collection_cache[collection_href] = collection return properties_merged
def copy_command(src: str, dst: str, catalog_type: pystac.CatalogType, copy_assets: bool, publish_location: Optional[str]) -> None: """Copy a STAC Catalog or Collection at SRC to the directory at DST. Note: Copying a catalog will upgrade it to the latest version of STAC.""" source_catalog = pystac.read_file(make_absolute_href(src)) if not isinstance(source_catalog, pystac.Catalog): raise click.BadArgumentUsage(f"{src} is not a STAC Catalog") copy_catalog(source_catalog, dst, catalog_type, copy_assets, publish_location)
def copy_catalog(source_catalog, dest_directory, catalog_type=None, copy_assets=False): catalog = source_catalog.full_copy() dest_directory = make_absolute_href(dest_directory) catalog.normalize_hrefs(dest_directory) if copy_assets: catalog = move_all_assets(catalog, copy=True, make_hrefs_relative=True) catalog.save(catalog_type)
def validate_catalog_link_type( href: str, link_type: str, should_include_self: bool ) -> None: cat_dict = pystac.StacIO.default().read_json(href) cat = pystac.read_file(href) assert isinstance(cat, pystac.Catalog) rels = set([link["rel"] for link in cat_dict["links"]]) self.assertEqual("self" in rels, should_include_self) for child_link in cat.get_child_links(): child_href = make_absolute_href(child_link.href, href) validate_catalog_link_type( child_href, link_type, catalog_type == CatalogType.ABSOLUTE_PUBLISHED, ) for item_link in cat.get_item_links(): item_href = make_absolute_href(item_link.href, href) validate_item_link_type( item_href, link_type, catalog_type == CatalogType.ABSOLUTE_PUBLISHED )
def validate_all(stac_dict: Dict[str, Any], href: str, stac_io: Optional[pystac.StacIO] = None) -> None: """Validate STAC JSON and all contained catalogs, collections and items. If this stac_dict represents a catalog or collection, this method will recursively be called for each child link and all contained items. Args: stac_dict : Dictionary that is the STAC json of the object. href : HREF of the STAC object being validated. Used for error reporting and resolving relative links. stac_io: Optional StacIO instance to use for reading hrefs. If None, the StacIO.default() instance is used. Raises: STACValidationError: This will raise a STACValidationError if this or any contained catalog, collection or item has a validation error. """ if stac_io is None: stac_io = pystac.StacIO.default() info = identify_stac_object(stac_dict) # Validate this object validate_dict( stac_dict, stac_object_type=info.object_type, stac_version=str(info.version_range.latest_valid_version()), extensions=list(info.extensions), href=href, ) if info.object_type != pystac.STACObjectType.ITEM: if "links" in stac_dict: # Account for 0.6 links if isinstance(stac_dict["links"], dict): links: List[Dict[str, Any]] = list(stac_dict["links"].values()) else: links = cast(List[Dict[str, Any]], stac_dict.get("links")) for link in links: rel = link.get("rel") if rel in [pystac.RelType.ITEM, pystac.RelType.CHILD]: link_href = make_absolute_href(cast(str, link.get("href")), start_href=href) if link_href is not None: d = stac_io.read_json(link_href) validate_all(d, link_href)
def test_add_raster_to_item(self): with TemporaryDirectory() as tmp_dir: catalog = create_temp_catalog_copy(tmp_dir) items = list(catalog.get_all_items()) item_path = make_absolute_href(items[0].get_self_href(), catalog.get_self_href()) cmd = ["addraster", item_path] self.run_command(cmd) updated = pystac.read_file(catalog.get_self_href()) item = list(updated.get_all_items())[0] asset = item.get_assets().get("analytic") assert asset is not None expected = expected_json("rasterbands.json") assert expected == asset.to_dict().get("raster:bands")
def get_absolute_href(self): """Gets the absolute href for this asset, if possible. If this Asset has no associated Item, this will return whatever the href is (as it cannot determine the absolute path, if the asset href is relative). Returns: str: The absolute HREF of this asset, or a relative HREF is an abslolute HREF cannot be determined. """ if not is_absolute_href(self.href): if self.owner is not None: return make_absolute_href(self.href, self.owner.get_self_href()) return self.href
def get_absolute_href(self): """Gets the absolute href for this link, if possible. Returns: str: Returns this link's HREF. It attempts to derive an absolute HREF from this link; however, if the link is relative, has no owner, and has an unresolved target, this will return a relative HREF. """ if self.is_resolved(): href = self.target.get_self_href() else: href = self.target if self.owner is not None: href = make_absolute_href(href, self.owner.get_self_href()) return href
def copy_catalog(source_catalog: Catalog, dest_directory: str, catalog_type: Optional[CatalogType] = None, copy_assets: bool = False, publish_location: Optional[str] = None) -> None: catalog = source_catalog.full_copy() dest_directory = make_absolute_href(dest_directory) if copy_assets: catalog = move_all_assets(catalog, copy=True, make_hrefs_relative=True) if publish_location is not None: catalog.normalize_hrefs(publish_location) catalog.save(catalog_type, dest_directory) else: catalog.normalize_hrefs(dest_directory) catalog.save(catalog_type)
def get_absolute_href(self) -> Optional[str]: """Gets the absolute href for this asset, if possible. If this Asset has no associated Item, and the asset HREF is a relative path, this method will return None. Returns: str: The absolute HREF of this asset, or None if an absolute HREF could not be determined. """ if utils.is_absolute_href(self.href): return self.href else: if self.owner is not None: return utils.make_absolute_href(self.href, self.owner.get_self_href()) else: return None
def from_file(cls, href: str, stac_io: Optional[pystac.StacIO] = None) -> "ItemCollection": """Reads a :class:`ItemCollection` from a JSON file. Arguments: href : Path to the file. stac_io : A :class:`~pystac.StacIO` instance to use for file I/O """ if stac_io is None: stac_io = pystac.StacIO.default() if not is_absolute_href(href): href = make_absolute_href(href) d = stac_io.read_json(href) return cls.from_dict(d, preserve_dict=False)
def test_make_absolute_href(self): # Test cases of (source_href, start_href, expected) test_cases = [('item.json', '/a/b/c/catalog.json', '/a/b/c/item.json'), ('./item.json', '/a/b/c/catalog.json', '/a/b/c/item.json'), ('./z/item.json', '/a/b/c/catalog.json', '/a/b/c/z/item.json'), ('../item.json', '/a/b/c/catalog.json', '/a/b/item.json'), ('item.json', 'https://stacspec.org/a/b/c/catalog.json', 'https://stacspec.org/a/b/c/item.json'), ('./item.json', 'https://stacspec.org/a/b/c/catalog.json', 'https://stacspec.org/a/b/c/item.json'), ('./z/item.json', 'https://stacspec.org/a/b/c/catalog.json', 'https://stacspec.org/a/b/c/z/item.json'), ('../item.json', 'https://stacspec.org/a/b/c/catalog.json', 'https://stacspec.org/a/b/item.json')] for source_href, start_href, expected in test_cases: actual = make_absolute_href(source_href, start_href) self.assertEqual(actual, expected)
def resolve_stac_object(self, root=None): """Resolves a STAC object from the HREF of this link, if the link is not already resolved. Args: root (Catalog or Collection): Optional root of the catalog for this link. If provided, the root's resolved object cache is used to search for previously resolved instances of the STAC object. """ if isinstance(self.target, str): target_href = self.target # If it's a relative link, base it off the parent. if not is_absolute_href(target_href): if self.owner is None: raise STACError('Relative path {} encountered ' 'without owner or start_href.'.format(target_href)) start_href = self.owner.get_self_href() if start_href is None: raise STACError('Relative path {} encountered ' 'without owner "self" link set.'.format(target_href)) target_href = make_absolute_href(target_href, start_href) obj = None if root is not None: obj = root._resolved_objects.get_by_href(target_href) if obj is None: obj = STAC_IO.read_stac_object(target_href, root=root) obj.set_self_href(target_href) if root is not None: obj = root._resolved_objects.get_or_cache(obj) obj.set_root(root, link_type=self.link_type) else: obj = self.target self.target = obj if self.owner and self.rel in ['child', 'item']: self.target.set_parent(self.owner, link_type=self.link_type) return self
def test_make_absolute_href_windows(self) -> None: utils._pathlib = ntpath try: # Test cases of (source_href, start_href, expected) test_cases = [ ("item.json", "C:\\a\\b\\c\\catalog.json", "C:\\a\\b\\c\\item.json"), (".\\item.json", "C:\\a\\b\\c\\catalog.json", "C:\\a\\b\\c\\item.json"), ( ".\\z\\item.json", "Z:\\a\\b\\c\\catalog.json", "Z:\\a\\b\\c\\z\\item.json", ), ("..\\item.json", "a:\\a\\b\\c\\catalog.json", "a:\\a\\b\\item.json"), ( "item.json", "HTTPS://stacspec.org/a/b/c/catalog.json", "https://stacspec.org/a/b/c/item.json", ), ( "./item.json", "https://stacspec.org/a/b/c/catalog.json", "https://stacspec.org/a/b/c/item.json", ), ( "./z/item.json", "https://stacspec.org/a/b/c/catalog.json", "https://stacspec.org/a/b/c/z/item.json", ), ( "../item.json", "https://stacspec.org/a/b/c/catalog.json", "https://stacspec.org/a/b/item.json", ), ] for source_href, start_href, expected in test_cases: actual = make_absolute_href(source_href, start_href) self.assertEqual(actual, expected) finally: utils._pathlib = os.path
def create_cog_asset(key, asset, path): asset_filename, extension = os.path.splitext(os.path.split(asset.href)[1]) cog_filename = f'{asset_filename}-COG.tif' cog_path = os.path.join(path, cog_filename) with TemporaryDirectory() as tmp_dir: reprojected_path = os.path.join(tmp_dir, f'reprojected{extension}') print(f'reprojecting {asset.href}') reproject(asset.href, reprojected_path) print(f'cogifying {asset.href}') cogify(reprojected_path, cog_path) shutil.rmtree(tmp_dir, ignore_errors=True) asset = pystac.Asset(href=make_absolute_href(cog_path), media_type=pystac.MediaType.COG, roles=['data'], title=f'{asset.title} (COG)', properties=asset.properties) return (f'{key}-cog', asset)
def add_raster_to_item(item: Item) -> Item: """Adds raster extension values to an item. Args: item (Item): The PySTAC Item to extend. Returns: Item: Returns an updated Item. This operation mutates the Item. """ RasterExtension.add_to(item) for asset in item.assets.values(): if asset.roles and "data" in asset.roles: raster = RasterExtension.ext(asset) href = make_absolute_href(asset.href, item.get_self_href()) bands = _read_bands(href) if bands: raster.apply(bands) return item
def normalize_hrefs(self, root_href): # Normalizing requires an absolute path if not is_absolute_href(root_href): root_href = make_absolute_href(root_href, os.getcwd(), start_is_dir=True) # Fully resolve the STAC to avoid linking issues. # This particularly can happen with unresolved links that have # relative paths. self.fully_resolve() for child in self.get_children(): child_root = os.path.join(root_href, '{}/'.format(child.id)) child.normalize_hrefs(child_root) for item in self.get_items(): item_root = os.path.join(root_href, '{}'.format(item.id)) item.normalize_hrefs(item_root) self.set_self_href(os.path.join(root_href, self.DEFAULT_FILE_NAME)) return self
def make_asset_hrefs_absolute(self): """Modify each asset's HREF to be absolute. Any asset HREFs that are relative will be modified to absolute based on this item's self HREF. Returns: Item: self """ self_href = None for asset in self.assets.values(): href = asset.href if not is_absolute_href(href): if self_href is None: self_href = self.get_self_href() if self_href is None: raise STACError('Cannot make relative asset HREFs absolute ' 'if no self_href is set.') asset.href = make_absolute_href(asset.href, self_href) return self
def validate_all(stac_dict, href): """Validate STAC JSON and all contained catalogs, collections and items. If this stac_dict represents a catalog or collection, this method will recursively be called for each child link and all contained items. Args: stac_dict (dict): Dictionary that is the STAC json of the object. href (str): HREF of the STAC object being validated. Used for error reporting and resolving relative links. Raises: STACValidationError: This will raise a STACValidationError if this or any contained catalog, collection or item has a validation error. """ info = identify_stac_object(stac_dict) # Validate this object validate_dict(stac_dict, stac_object_type=info.object_type, stac_version=info.version_range.latest_valid_version(), extensions=info.common_extensions, href=href) if info.object_type != pystac.STACObjectType.ITEM: links = stac_dict.get('links') if links is not None: # Account for 0.6 links if isinstance(links, dict): links = list(links.values()) for link in links: rel = link.get('rel') if rel in ['item', 'child']: link_href = make_absolute_href(link.get('href'), start_href=href) if link_href is not None: d = pystac.STAC_IO.read_json(link_href) validate_all(d, link_href)
def __init__( self, rel: Union[str, pystac.RelType], target: Union[str, "STACObject_Type"], media_type: Optional[str] = None, title: Optional[str] = None, extra_fields: Optional[Dict[str, Any]] = None, ) -> None: self.rel = rel if isinstance(target, str): if rel == pystac.RelType.SELF: self._target_href = make_absolute_href(target) else: self._target_href = target self._target_object = None else: self._target_href = None self._target_object = target self.media_type = media_type self.title = title self.extra_fields = extra_fields or {} self.owner = None
def add_raster_to_item(item: Item) -> Item: """Adds raster extension values to an item. Args: item (Item): The PySTAC Item to extend. Returns: Item: Returns an updated Item. This operation mutates the Item. """ RasterExtension.add_to(item) for asset in item.assets.values(): if asset.roles and "data" in asset.roles: raster = RasterExtension.ext(asset) bands = [] href = make_absolute_href(asset.href, item.get_self_href()) dataset = gdal.Open(href, gdal.GA_ReadOnly) for nband in range(dataset.RasterCount): gdal_band = dataset.GetRasterBand(nband + 1) band = RasterBand.create() band.nodata = gdal_band.GetNoDataValue() band.spatial_resolution = dataset.GetGeoTransform()[1] band.data_type = DataType( gdal.GetDataTypeName(gdal_band.DataType).lower()) minimum = gdal_band.GetMinimum() maximum = gdal_band.GetMaximum() if not minimum or not max: minimum, maximum = gdal_band.ComputeRasterMinMax(True) band.statistics = Statistics.create(minimum=minimum, maximum=maximum) hist_data = gdal_band.GetHistogram(minimum, maximum, NUM_BUCKETS) band.histogram = Histogram.create(NUM_BUCKETS, minimum, maximum, hist_data) bands.append(band) if bands: raster.apply(bands) return item