def test_migrate(self):
collection_cache = CollectionCache()
for example in self.examples:
path = example['path']
d = STAC_IO.read_json(path)
if identify_stac_object_type(d) == STACObjectType.ITEM:
merge_common_properties(d,
json_href=path,
collection_cache=collection_cache)
info = identify_stac_object(d)
migrated_d = migrate_to_latest(d, info)
migrated_info = identify_stac_object(migrated_d)
self.assertEqual(migrated_info.object_type, info.object_type)
self.assertEqual(
migrated_info.version_range.latest_valid_version(),
STAC_VERSION)
self.assertEqual(set(migrated_info.common_extensions),
set(info.common_extensions))
self.assertEqual(set(migrated_info.custom_extensions),
set(info.custom_extensions))
# Test that PySTAC can read it without errors.
self.assertIsInstance(
STAC_IO.stac_object_from_dict(migrated_d, href=path),
STACObject)
def _setup():
if not SchemaValidator._is_setup:
for c in SchemaValidator.core_schemas:
uri = SchemaValidator._abs_schema(
SchemaValidator.core_schemas[c])
SchemaValidator.core_schemas[c] = uri
SchemaValidator._schema_cache[uri] = json.loads(
STAC_IO.read_text(uri))
for ext in SchemaValidator.extension_schemas:
for stac_object_type in SchemaValidator.extension_schemas[ext]:
uri = SchemaValidator._abs_schema(
SchemaValidator.extension_schemas[ext]
[stac_object_type])
SchemaValidator.extension_schemas[ext][
stac_object_type] = uri
SchemaValidator._schema_cache[uri] = json.loads(
STAC_IO.read_text(uri))
for uri in SchemaValidator.aux_schemas:
abs_uri = SchemaValidator._abs_schema(uri)
SchemaValidator._schema_cache[abs_uri] = json.loads(
STAC_IO.read_text(abs_uri))
SchemaValidator._is_setup = True
def s3stac_write(uri, txt):
extra = {
'ContentType': 'application/json'
}
if uri.startswith('s3'):
s3().upload_json(json.loads(txt), uri, extra=extra, public=PUBLIC_CATALOG)
else:
STAC_IO.default_write_text_method(uri, txt)
def my_write_method(uri, txt):
parsed = urlparse(uri)
if parsed.scheme == 's3':
bucket = parsed.netloc
key = parsed.path[1:]
s3 = boto3.resource("s3")
s3.Object(bucket, key).put(Body=txt)
else:
STAC_IO.default_write_text_method(uri, txt)
def s3_write(uri, txt):
parsed = urlparse(uri)
if parsed.scheme == "s3":
bucket = parsed.netloc
key = parsed.path.lstrip("/")
s3 = boto3.resource("s3")
s3.Object(bucket, key).put(Body=txt)
else:
STAC_IO.default_write_text_method(uri, txt)
def write_remote_stacs(uri, txt):
"""
Writes STACs from a remote location. To be used to set STAC_IO
Defaults to local storage.
"""
parsed = urlparse(uri)
if parsed.scheme == "s3":
bucket = parsed.netloc
key = parsed.path[1:]
s3 = boto3.resource("s3")
s3.Object(bucket, key).put(Body=txt)
else:
STAC_IO.default_write_text_method(uri, txt)
def test_from_file_pre_081(self):
d = STAC_IO.read_json(self.label_example_1_uri)
d['stac_version'] = '0.8.0-rc1'
d['properties']['label:property'] = d['properties']['label:properties']
d['properties'].pop('label:properties')
d['properties']['label:overview'] = d['properties']['label:overviews']
d['properties'].pop('label:overviews')
d['properties']['label:method'] = d['properties']['label:methods']
d['properties'].pop('label:methods')
d['properties']['label:task'] = d['properties']['label:tasks']
d['properties'].pop('label:tasks')
label_example_1 = STAC_IO.stac_object_from_dict(d)
self.assertEqual(len(label_example_1.ext.label.label_tasks), 2)
def test_identify(self):
collection_cache = {}
for example in self.examples:
path = example['path']
d = STAC_IO.read_json(path)
actual = identify_stac_object(d,
merge_collection_properties=True,
json_href=path,
collection_cache=collection_cache)
msg = 'Failed {}:'.format(path)
self.assertEqual(actual.object_type,
example['object_type'],
msg=msg)
version_contained_in_range = actual.version_range.contains(
example['stac_version'])
self.assertTrue(version_contained_in_range, msg=msg)
self.assertEqual(set(actual.common_extensions),
set(example['common_extensions']),
msg=msg)
self.assertEqual(set(actual.custom_extensions),
set(example['custom_extensions']),
msg=msg)
def paginate(
request: Request,
next_resolver: Callable,
) -> Iterator[dict]:
"""
Parameters
----------
request : urllib.request.Request
The initial request to start paging. Subsequent requests will be determined by the ``next_resolver``.
next_resolver : Callable
An callable that will be used to construct the request for the next page of results based on the ``"next"``
link from the previous page.
"""
while True:
# Yield all items
page = STAC_IO.read_json(request)
yield page
# Get the next link and make the next request
next_link = next(
(link for link in page.get('links', []) if link['rel'] == 'next'),
None)
if next_link is None:
break
request = next_resolver(next_link, request)
def download_metadata_command(destination, id, quiet):
"""Creates a 3DEP collection in DESTINATION."""
base_ids = id # not sure how to rename arguments in click
for product in PRODUCTS:
if base_ids:
ids = base_ids
else:
ids = utils.fetch_ids(product)
for id in ids:
path = utils.path(product,
id,
extension="xml",
base=destination)
if os.path.exists(path):
if not quiet:
print("{} exists, skipping download...".format(path))
continue
os.makedirs(os.path.dirname(path), exist_ok=True)
source_path = utils.path(product,
id,
extension="xml",
base=USGS_FTP_BASE)
if not quiet:
print("{} -> {}".format(source_path, path))
text = STAC_IO.read_text(source_path)
with open(path, "w") as f:
f.write(text)
def test_identify(self):
collection_cache = CollectionCache()
for example in self.examples:
path = example['path']
d = STAC_IO.read_json(path)
if identify_stac_object_type(d) == STACObjectType.ITEM:
try:
merge_common_properties(d, json_href=path, collection_cache=collection_cache)
except HTTPError:
pass
actual = identify_stac_object(d)
# Explicitly cover __repr__ functions in tests
str_info = str(actual)
self.assertIsInstance(str_info, str)
msg = 'Failed {}:'.format(path)
self.assertEqual(actual.object_type, example['object_type'], msg=msg)
version_contained_in_range = actual.version_range.contains(example['stac_version'])
self.assertTrue(version_contained_in_range, msg=msg)
self.assertEqual(set(actual.common_extensions),
set(example['common_extensions']),
msg=msg)
self.assertEqual(set(actual.custom_extensions),
set(example['custom_extensions']),
msg=msg)
def validate_item_link_type(href, link_type, should_include_self):
item_dict = STAC_IO.read_json(href)
item = STACObject.from_file(href)
for link in item.get_links():
if not link.rel == 'self':
self.assertEqual(link.link_type, link_type)
rels = set([link['rel'] for link in item_dict['links']])
self.assertEqual('self' in rels, should_include_self)
def s3_read(uri):
parsed = urlparse(uri)
if parsed.scheme == "s3":
bucket = parsed.netloc
key = parsed.path.lstrip("/")
s3 = boto3.resource("s3")
obj = s3.Object(bucket, key)
return obj.get()["Body"].read().decode("utf-8")
else:
return STAC_IO.default_read_text_method(uri)
def s3_read(uri):
parsed = urlparse(uri)
if parsed.scheme == 's3':
bucket = parsed.netloc
key = parsed.path[1:]
s3 = boto3.resource('s3')
obj = s3.Object(bucket, key)
return obj.get()['Body'].read().decode('utf-8')
else:
return STAC_IO.default_read_text_method(uri)
def get_schema_from_uri(self, schema_uri):
if schema_uri not in self.schema_cache:
s = json.loads(STAC_IO.read_text(schema_uri))
self.schema_cache[schema_uri] = s
schema = self.schema_cache[schema_uri]
resolver = jsonschema.validators.RefResolver(base_uri=schema_uri,
referrer=schema,
store=self.schema_cache)
return (schema, resolver)
def get_schema(self, obj_type):
schema_uri = SchemaValidator.schemas.get(obj_type)
if schema_uri is None:
raise Exception('No schema for type {}'.format(obj_type))
schema = self.schema_cache.get(obj_type)
if schema is None:
schema = json.loads(STAC_IO.read_text(schema_uri))
self.schema_cache[obj_type] = schema
resolver = RefResolver(base_uri=schema_uri, referrer=schema)
return (schema, resolver)
def test_from_file_pre_081(self):
d = STAC_IO.read_json(self.label_example_1_uri)
d['properties']['label:property'] = d['properties']['label:properties']
d['properties'].pop('label:properties')
d['properties']['label:overview'] = d['properties']['label:overviews']
d['properties'].pop('label:overviews')
d['properties']['label:method'] = d['properties']['label:methods']
d['properties'].pop('label:methods')
d['properties']['label:task'] = d['properties']['label:tasks']
d['properties'].pop('label:tasks')
label_example_1 = LabelItem.from_dict(d)
self.assertEqual(len(label_example_1.label_tasks), 1)
def handle_dataset(version_metadata_key: str) -> None:
"""Handle writing a new dataset version to the dataset catalog"""
storage_bucket_path = f"{S3_URL_PREFIX}{ResourceName.STORAGE_BUCKET_NAME.value}"
dataset_prefix = version_metadata_key.split("/", maxsplit=1)[0]
dataset_catalog = Catalog.from_file(
f"{storage_bucket_path}/{dataset_prefix}/{CATALOG_KEY}")
dataset_version_metadata = STAC_IO.read_stac_object(
f"{storage_bucket_path}/{version_metadata_key}")
dataset_catalog.add_child(dataset_version_metadata,
strategy=GeostoreSTACLayoutStrategy())
dataset_catalog.normalize_hrefs(f"{storage_bucket_path}/{dataset_prefix}",
strategy=GeostoreSTACLayoutStrategy())
dataset_catalog.save(catalog_type=CatalogType.SELF_CONTAINED)
def read_text_method(uri):
"""Overwrites the default method for reading text from a URL or file to allow :class:`urllib.request.Request`
instances as input. This method also raises any :exc:`urllib.error.HTTPError` exceptions rather than catching
them to allow us to handle different response status codes as needed."""
if isinstance(uri, Request):
logger.debug(
f"Requesting {uri.get_full_url()} with headers {uri.headers}")
with urlopen(uri) as response:
resp = response.read()
return resp.decode("utf-8")
elif bool(urlparse(uri).scheme):
logger.debug(f"Requesting {uri}")
resp = requests.get(uri)
return resp.content.decode("utf-8")
else:
return STAC_IO.default_read_text_method(uri)
def my_read_method(uri):
parsed = urlparse(uri)
if parsed.scheme.startswith('http'):
if os.environ.get('STAGEIN_PASSWORD') is None:
return requests.get(uri).text
else:
return requests.get(uri,
auth=HTTPBasicAuth(os.environ.get('STAGEIN_USERNAME'),
os.environ.get('STAGEIN_PASSWORD'))
).text
else:
return STAC_IO.default_read_text_method(uri)
def read_remote_stacs(uri):
"""
Reads STACs from a remote location. To be used to set STAC_IO
Defaults to local storage.
"""
parsed = urlparse(uri)
if parsed.scheme == "s3":
bucket = parsed.netloc
key = parsed.path[1:]
s3 = boto3.resource("s3")
obj = s3.Object(bucket, key)
return json.loads(obj.get()["Body"].read().decode("utf-8"))
if parsed.scheme in ["http", "https"]:
with urllib.request.urlopen(uri) as url:
stac = json.loads(url.read().decode())
return stac
else:
return STAC_IO.default_read_text_method(uri)
def remove_bad_collection(js):
links = js.get('links')
if links is not None:
filtered_links = []
for link in links:
rel = link.get('rel')
if rel is not None and rel == 'collection':
href = link['href']
try:
json.loads(STAC_IO.read_text(href))
filtered_links.append(link)
except (HTTPError, FileNotFoundError,
json.decoder.JSONDecodeError):
print(
'===REMOVING UNREADABLE COLLECTION AT {}'.format(href))
else:
filtered_links.append(link)
js['links'] = filtered_links
return js
def validate_catalog_link_type(href, link_type, should_include_self):
cat_dict = STAC_IO.read_json(href)
cat = STACObject.from_file(href)
for link in cat.get_links():
if not link.rel == 'self':
self.assertEqual(link.link_type, link_type)
rels = set([link['rel'] for link in cat_dict['links']])
self.assertEqual('self' in rels, should_include_self)
for child_link in cat.get_child_links():
child_href = make_absolute_href(child_link.target, href)
validate_catalog_link_type(child_href, link_type,
catalog_type == CatalogType.ABSOLUTE_PUBLISHED)
for item_link in cat.get_item_links():
item_href = make_absolute_href(item_link.target, href)
validate_item_link_type(item_href, link_type,
catalog_type == CatalogType.ABSOLUTE_PUBLISHED)
def validate_file(self, path, object_type):
d = STAC_IO.read_json(path)
return validate_dict(d, object_type)
def validate_file(self, path, object_type):
d = STAC_IO.read_json(path)
return self.schema_validator.validate_dict(d, object_type, print_on_error=True)
def s3stac_read(uri):
if uri.startswith('s3'):
return json.dumps(s3().read_json(uri))
else:
return STAC_IO.default_read_text_method(uri)
def my_read_method(uri):
parsed = urlparse(uri)
if parsed.scheme.startswith('http'):
return requests.get(uri).text
else:
return STAC_IO.default_read_text_method(uri)
