def collection2product_cli(collection, instrument, type, code, format, units, url, stac_version, outfile, ignore,
pre_collection,
verbose, access_token):
constants = {
'instrument_type': instrument,
'metadata_type': type,
'platform_code': code,
'format_name': format,
'units': units,
'ignore': ignore,
"is_pre_collection": pre_collection,
'verbose': verbose
}
_mapper = STAC_MAPPER_VERSIONS[stac_version]
access_token = _check_token(access_token)
c = stac.STAC(url, validate=False, access_token=access_token).collection(collection)
yaml_content = stac2odc.collection.collection2product(c, _mapper(), **constants)
if outfile is None:
print(yaml.dump(yaml_content))
else:
with open(outfile, 'w') as f:
yaml.dump(yaml_content, f)
def test_items(self, stac_objects, requests_mock, runner):
for k in stac_objects:
s = stac.STAC(url, True)
requests_mock.get(re.compile(url + '/stac'),
json=stac_objects[k]['catalog.json'],
status_code=200,
headers={'content-type': 'application/json'})
requests_mock.get(re.compile(url + '/collections'),
json=stac_objects[k]['collection.json'],
status_code=200,
headers={'content-type': 'application/json'})
requests_mock.get(re.compile(url +
'/collections/my_collection1/items'),
json=stac_objects[k]['items.json'],
status_code=200,
headers={'content-type': 'application/json'})
result = runner.invoke(stac.cli.items, [
'--url', url, '--collection-id', 'my_collection1', '--limit',
1, '--page', 1, '--datetime', '2016-05-03/2019-01-01',
'--bbox', '-180,-90,180,90'
])
assert result.exit_code == 0
assert 'feature1' in result.output
def item2dataset_cli(collection, instrument, code, format, units, url, stac_version, basepath, outpath, ignore,
max_items,
pre_collection, verbose, download, download_out, advanced_filter, access_token):
constants = {
'instrument_type': instrument,
'plataform_code': code,
'format_name': format,
'units': units,
'basepath': basepath,
'ignore': ignore,
'outpath': outpath,
'max_items': int(max_items),
"is_pre_collection": pre_collection,
'verbose': verbose,
"download": download,
"download_out": download_out
}
mapper = STAC_MAPPER_VERSIONS[stac_version]
_filter = {"collections": [collection]}
if advanced_filter:
_filter = {
**_filter, **utils.prepare_advanced_filter(advanced_filter)
}
access_token = _check_token(access_token)
s = stac.STAC(url, validate=False, access_token=access_token)
stac2odc.item.item2dataset(s, _filter, mapper(), **constants)
def test_search(self, stac_objects, requests_mock, runner):
for k in stac_objects:
s = stac.STAC(url, True)
requests_mock.get(re.compile(url + '/stac'),
json=stac_objects[k]['catalog.json'],
status_code=200,
headers={'content-type': 'application/json'})
requests_mock.get(re.compile(url + '/collections'),
json=stac_objects[k]['collection.json'],
status_code=200,
headers={'content-type': 'application/json'})
requests_mock.post(re.compile(url + '/stac/search'),
json=stac_objects[k]['items.json'],
status_code=200,
headers={'content-type': 'application/json'})
intersects = json.dumps(
stac_objects[k]['items.json']['features'][0]['geometry'])
result = runner.invoke(stac.cli.search, [
'--url', url, '--collections', 'my_collection1', '--ids',
'feature1', '--intersects', intersects, '--limit', 1, '--next',
'aaa', '--page', 1, '--datetime', '2016-05-03/2019-01-01',
'--bbox', '-180,-90,180,90'
])
assert result.exit_code == 0
assert 'feature1' in result.output
def describe(collection_name):
"""Get a description with available collection name from STAC.
## Parameters
### collection_name : string, required
The collection name from list collections.
### token : string, required
The BDC user token.
## Raise
### HTTPError
If the STAC server is out of service.
"""
token = request.args.get('token')
try:
service = stac.STAC(config.STAC_URL, access_token=token)
return jsonify(service.collections[collection_name])
except:
return jsonify({'code': '403', 'message': 'Access error, forbidden!'})
def stac2json_cli(bdc_stac_search_bbox: str, bdc_stac_search_limit: int,
bdc_stac_search_datetime: str, bdc_stac_url: str,
bdc_stac_collection: str, bdc_access_token: str,
output_file: str):
#
# client instance
#
client = stac.STAC(bdc_stac_url, access_token=bdc_access_token)
#
# get stac-features
#
collection = client.collection(bdc_stac_collection)
# west, south, east, north
items = collection.get_items(
filter={
"bbox": bdc_stac_search_bbox,
"datetime": bdc_stac_search_datetime,
"limit": bdc_stac_search_limit
})
#
# export stac-features
#
with open(output_file, "w") as f:
json.dump(items, f)
def test_collection(self, stac_objects, requests_mock):
for k in stac_objects:
s = stac.STAC(url, True)
requests_mock.get(match_url,
json=stac_objects[k]['catalog.json'],
status_code=200,
headers={'content-type': 'application/json'})
response = s.catalog
requests_mock.get(match_url,
json=stac_objects[k]['collection.json'],
status_code=200,
headers={'content-type': 'application/json'})
collection = s.collection('my_collection1')
assert collection == stac_objects[k]['collection.json']
assert collection.keywords
assert collection.version
assert collection.license
assert collection.properties
assert collection.providers[0].name
assert collection.providers[0].description
assert collection.providers[0].roles
assert collection.providers[0].url
if k in ['0.8.0', '0.8.1']:
assert collection.extent.spatial.bbox
assert collection.extent.temporal.interval
assert collection.summaries
assert collection.summaries['val'].min == 0
assert collection.summaries['val'].max == 1
else:
assert collection.extent.spatial
assert collection.extent.temporal
def test_item(self, stac_objects, requests_mock):
for k in stac_objects:
s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True)
if k == '0.9.0':
requests_mock.get(re.compile(url + '/'),
json=stac_objects[k]['catalog.json'],
status_code=200,
headers={'content-type': 'application/json'})
else:
requests_mock.get(re.compile(url + '/stac'),
json=stac_objects[k]['catalog.json'],
status_code=200,
headers={'content-type': 'application/json'})
requests_mock.get(
re.compile(url + '/collections'),
json=dict(collections=[stac_objects[k]['collection.json']]),
status_code=200,
headers={'content-type': 'application/json'})
catalog = s.catalog
s.catalog #test for non empty catalog
requests_mock.get(match_url,
json=stac_objects[k]['collection.json'],
status_code=200,
headers={'content-type': 'application/json'})
collection = s.collection('my_collection1')
requests_mock.get(match_url,
json=stac_objects[k]['items.json'],
status_code=200,
headers={'content-type': 'application/json'})
itemcollection = s.collection('my_collection1').get_items()
item = itemcollection.features[0]
if collection.stac_version == '0.8.0':
assert itemcollection.links
assert item.properties.datetime
assert item.properties.license
assert item.properties.providers
assert item.properties.title
assert item.properties.created
assert item.properties.updated
assert itemcollection.type
assert item.id == 'feature1'
assert item.stac_version
assert item.type
assert item.bbox
assert item.collection
assert item.geometry.type
assert item.geometry.coordinates
assert item.properties
assert item.links
assert item.assets['thumbnail'].href
assert item.assets['thumbnail'].title
assert item.assets['thumbnail'].type
def test_search(self, stac_objects, requests_mock):
for k in stac_objects:
s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True)
requests_mock.get(match_url,
json=stac_objects[k]['catalog.json'],
status_code=200,
headers={'content-type': 'application/json'})
requests_mock.get(
re.compile(url + '/collections'),
json=dict(collections=[stac_objects[k]['collection.json']]),
status_code=200,
headers={'content-type': 'application/json'})
response = s.catalog
requests_mock.get(match_url,
json=stac_objects[k]['items.json'],
status_code=200,
headers={'content-type': 'application/json'})
response = s.search()
assert response.features[0].id == 'feature1'
def test_catalog(self, stac_objects, requests_mock):
for k in stac_objects:
s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True)
requests_mock.get(match_url,
json=stac_objects[k]['catalog.json'],
status_code=200,
headers={'content-type': 'application/json'})
requests_mock.get(
re.compile(url + '/collections'),
json=dict(collections=[stac_objects[k]['collection.json']]),
status_code=200,
headers={'content-type': 'application/json'})
response = s.catalog
assert s._catalog.stac_version
assert s._catalog.id
assert s._catalog.description
assert s._catalog.title
assert s._catalog.links[0].type
assert s._catalog.links[0].title
assert s._catalog.links[0].href
assert s._catalog.links[0].rel
assert response == ['my_collection1']
def item2dataset_cli(stac_collection, dc_product, url, outdir, max_items,
engine_file, datacube_config, verbose, access_token,
advanced_filter):
_filter = {"collections": [stac_collection]}
if advanced_filter:
_filter = {**_filter, **prepare_advanced_filter(advanced_filter)}
stac_service = stac.STAC(url, False, access_token=access_token)
dc_index = datacube_index(datacube_config)
features = create_feature_collection_from_stac_elements(
stac_service, int(max_items), _filter)
odc_datasets = stac2odc.item.item2dataset(engine_file,
dc_product,
features,
dc_index,
verbose=verbose)
odc_datasets_definition_files = write_odc_element_in_yaml_file(
odc_datasets, outdir)
# add datasets definitions on datacube index
# code adapted from: https://github.com/opendatacube/datacube-core/blob/develop/datacube/scripts/dataset.py
ds_resolve = Doc2Dataset(dc_index, [dc_product])
doc_stream = remap_uri_from_doc(
ui_path_doc_stream(odc_datasets_definition_files, uri=True))
datasets_on_stream = dataset_stream(doc_stream, ds_resolve)
logger_message(f"Adding datasets", logger.info, True)
for dataset in datasets_on_stream:
try:
dc_index.datasets.add(dataset, with_lineage=True)
except (ValueError, MissingRecordError):
logger_message(f"Error to add dataset ({dataset.local_uri})",
logger.warning, True)
def test_collection_without_catalog(self, stac_objects, requests_mock):
for k in stac_objects:
s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True)
requests_mock.get(match_url, json=stac_objects[k]['collection.json'],
status_code=200,
headers={'content-type':'application/json'})
response = s.collection('my_collection1')
assert response == stac_objects[k]['collection.json']
def test_catalog(self, stac_objects, requests_mock, runner):
for k in stac_objects:
s = stac.STAC(url, True)
requests_mock.get(match_url,
json=stac_objects[k]['catalog.json'],
status_code=200,
headers={'content-type': 'application/json'})
result = runner.invoke(stac.cli.catalog, ['--url', url])
assert result.exit_code == 0
assert 'my_collection1' in result.output
def test_search(self, stac_objects, requests_mock):
for k in stac_objects:
s = stac.STAC(url, True)
requests_mock.get(match_url,
json=stac_objects[k]['items.json'],
status_code=200,
headers={'content-type': 'application/json'})
response = s.search()
assert response.features[0].id == 'feature1'
def test_item_empty(self, stac_objects, requests_mock):
for k in stac_objects:
s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True)
requests_mock.get(match_url, json=stac_objects[k]['collection.json'],
status_code=200,
headers={'content-type':'application/json'})
collection = s.collection('my_collection1')
requests_mock.get(match_url, json=stac_objects[k]['items.json']['features'][0],
status_code=200,
headers={'content-type':'application/json'})
collection['links'].pop(1)
assert collection.get_items() == stac.ItemCollection({})
def test_collection_missing(self, stac_objects, requests_mock):
for k in stac_objects:
s = stac.STAC(url, True)
requests_mock.get(match_url,
json=stac_objects[k]['catalog.json'],
status_code=200,
headers={'content-type': 'application/json'})
response = s.catalog
requests_mock.get(match_url, exc=requests.exceptions.HTTPError)
with pytest.raises(KeyError):
s.collection('missing_collection')
def search():
"""Get a description with available collection name from STAC.
## Parameters
### token : string, required
The BDC user token.
## JSON Object
### Test data for POST:
{
"collections": ["CB4_64_16D_STK-1"],
"bbox": [-46.01348876953125, -23.08478515994374, -45.703125, -23.34856015148709],
"interval": ["2018-08-01","2019-07-31"],
"limit": 10
}
## Raise
### HTTPError
If the STAC server is out of service.
"""
token = request.args.get('token')
if request.method == 'POST':
try:
collections = request.json['collections']
bbox = tuple(request.json['bbox'])
start_date = request.json['interval'][0]
end_date = request.json['interval'][1]
limit = request.json['limit']
try:
query = {
'collections': collections,
'bbox': bbox,
'datetime': f'{start_date}/{end_date}',
'limit': limit
}
items = stac.STAC(config.STAC_URL,
access_token=token).search(query)
return jsonify({"query": query, "result": items.features})
except:
return jsonify({
'code': '403',
'message': 'Access error, forbidden!'
})
except:
return jsonify({'code': '400', 'message': 'Request JSON error!'})
def __init__(self):
"""Build DataCube object with config parameters including access token, STAC url and earth observation service url."""
if len(config.ACCESS_TOKEN) == 0:
config.ACCESS_TOKEN = input("Please insert a valid user token from BDC Auth: ")
if len(config.EOCUBE_URL) == 0:
config.EOCUBE_URL = input("Please insert a valid url for EO Service: ")
if len(config.STAC_URL) == 0:
config.STAC_URL = input("Please insert a valid url for STAC Service: ")
self.stac_service = stac.STAC(
config.STAC_URL,
access_token=config.ACCESS_TOKEN
)
self.query = None
self.items = None
self.images = []
def test_stac_read_forbidden(self, stac_objects, requests_mock):
for k in stac_objects:
s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True)
requests_mock.get(match_url,
json=stac_objects[k]['catalog.json'],
status_code=200,
headers={'content-type': 'application/json'})
requests_mock.get(
re.compile(url + '/collections'),
json=dict(collections=[stac_objects[k]['collection.json']]),
status_code=200,
headers={'content-type': 'application/json'})
response = s.catalog
requests_mock.get(match_url,
json=stac_objects[k]['items.json'],
status_code=200,
headers={'content-type': 'application/json'})
response = s.search()
item = response.features[0]
requests_mock.head(item.assets['thumbnail']['href'],
status_code=403,
headers={'content-type': 'application/json'})
with pytest.raises(requests.exceptions.HTTPError) as err:
_ = item.read('thumbnail')
assert err.value.response.status_code == 403
# No reason is set when using Mock
assert str(err.value) == '(None) Missing Authentication Token.'
item.assets['thumbnail']['href'] += '?access_token=test'
requests_mock.head(item.assets['thumbnail']['href'],
status_code=403,
headers={'content-type': 'application/json'})
with pytest.raises(requests.exceptions.HTTPError) as err:
_ = item.read('thumbnail')
assert err.value.response.status_code == 403
assert str(
err.value
) == "(None) You don't have permission to request this resource."
def test_collection(self, stac_objects, requests_mock, runner):
for k in stac_objects:
s = stac.STAC(url, True)
requests_mock.get(re.compile(url + '/stac'),
json=stac_objects[k]['catalog.json'],
status_code=200,
headers={'content-type': 'application/json'})
requests_mock.get(re.compile(url + '/collections'),
json=stac_objects[k]['collection.json'],
status_code=200,
headers={'content-type': 'application/json'})
result = runner.invoke(
stac.cli.collection,
['--url', url, '--collection-id', 'my_collection1'])
assert result.exit_code == 0
assert 'my_collection1' in result.output
def item2dataset_cli(collection, instrument, code, format, units, url, basepath, outpath, ignore, max_items,
pre_collection, verbose, download, download_out):
constants = {
'instrument_type': instrument,
'plataform_code': code,
'format_name': format,
'units': units,
'basepath': basepath,
'ignore': ignore,
'outpath': outpath,
'max_items': int(max_items),
"is_pre_collection": pre_collection,
'verbose': verbose,
"download": download,
"download_out": download_out
}
s = stac.STAC(url, True)
c = s.collection(collection)
stac2odc.item.item2dataset(c, mapper.Stac2ODCMapper08(), **constants)
def test_item_id(self, stac_objects, requests_mock):
for k in stac_objects:
s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True)
requests_mock.get(match_url, json=stac_objects[k]['catalog.json'],
status_code=200,
headers={'content-type':'application/json'})
response = s.catalog
requests_mock.get(match_url, json=stac_objects[k]['collection.json'],
status_code=200,
headers={'content-type':'application/json'})
collection = s.collection('my_collection1')
requests_mock.get(match_url, json=stac_objects[k]['items.json']['features'][0],
status_code=200,
headers={'content-type':'application/json'})
response = collection.get_items(item_id='feature1')
assert response.id == 'feature1'
def test_empty_bbox(self, stac_objects, requests_mock):
for k in stac_objects:
s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True)
requests_mock.get(match_url,
json=stac_objects[k]['catalog.json'],
status_code=200,
headers={'content-type': 'application/json'})
requests_mock.get(
re.compile(url + '/collections'),
json=dict(collections=[stac_objects[k]['collection.json']]),
status_code=200,
headers={'content-type': 'application/json'})
with pytest.raises(TypeError):
s.search(bbox="")
with pytest.raises(TypeError):
s.search(bbox=-90)
def collection2product_cli(collection, instrument, type, code, format, units, url, outfile, ignore, pre_collection,
verbose):
constants = {
'instrument_type': instrument,
'metadata_type': type,
'platform_code': code,
'format_name': format,
'units': units,
'ignore': ignore,
"is_pre_collection": pre_collection,
'verbose': verbose
}
s = stac.STAC(url, True)
c = s.collection(collection)
yaml_content = stac2odc.collection.collection2product(c, mapper.Stac2ODCMapper08(), **constants)
if outfile is None:
print(yaml.dump(yaml_content))
else:
with open(outfile, 'w') as f:
yaml.dump(yaml_content, f)
def collections():
"""Get a list with available collections from STAC.
## Parameters
### token : string, optional
The BDC user token.
## Raise
### HTTPError
If the STAC server is out of service.
"""
token = request.args.get('token')
try:
service = stac.STAC(config.STAC_URL, access_token=token)
return jsonify({'collections': list(service.collections.keys())})
except:
return jsonify({'code': '403', 'message': 'Access error, forbidden!'})
def collection2product_cli(collection: str, url: str, outdir: str,
engine_file: str, datacube_config: str,
access_token, verbose: bool):
collection_definition = stac.STAC(
url, False, access_token=access_token).collection(collection)
odc_element = stac2odc.collection.collection2product(engine_file,
collection_definition,
verbose=verbose)
product_definition_file = write_odc_element_in_yaml_file(
odc_element, os.path.join(outdir, f'{collection}.yaml'))
# code adapted from: https://github.com/opendatacube/datacube-core/blob/develop/datacube/scripts/product.py
for path_descriptor, parsed_doc in read_documents(
*[product_definition_file]):
try:
dc_index = datacube_index(datacube_config)
_type = dc_index.products.from_doc(parsed_doc)
logger_message(f'Adding {_type.name}', logger.info, verbose)
dc_index.products.add(_type)
except InvalidDocException as e:
logger_message(f'Error to add product: {str(e)}', logger.warning,
True)
if __name__ == '__main__':
import stac
import yaml
import stac2odc.item
import stac2odc.collection
from stac2odc.mapper import Stac2ODCMapper09
constants = {
'instrument_type': 'AWFI',
'metadata_type': 'eo',
'platform_code': 'CBERS04',
'format_name': 'GeoTiff',
'units': 'm',
'ignore': ['quality'],
"is_pre_collection": False,
'verbose': True
}
outfile = 'test.yaml'
s = stac.STAC('http://brazildatacube.dpi.inpe.br/stac/', False)
c = s.collection('CB4_64_16D_STK-1')
yaml_content = stac2odc.collection.collection2product(
c, Stac2ODCMapper09(), **constants)
if outfile is None:
print(yaml.dump(yaml_content))
else:
with open(outfile, 'w') as f:
yaml.dump(yaml_content, f)
def test_stac(self):
s = stac.STAC(url, True)
assert s.url == url
assert repr(s) == f'stac("{url}")'
assert str(s) == f'<STAC [{url}]>'
if __name__ == '__main__':
import stac
import yaml
import stac2odc.item
import stac2odc.collection
from stac2odc.mapper import Stac2ODCMapper08
constants = {
'instrument_type': 'AWFI',
'metadata_type': 'eo',
'platform_code': 'CBERS04',
'format_name': 'GeoTiff',
'units': 'm',
'ignore': ['quality'],
"is_pre_collection": False,
'verbose': True
}
outfile = 'test.yaml'
s = stac.STAC('http://brazildatacube.dpi.inpe.br/bdc-stac/0.8.0/', True)
c = s.collection('CB4_64_16D_STK_v1')
yaml_content = stac2odc.collection.collection2product(
c, Stac2ODCMapper08(), **constants)
if outfile is None:
print(yaml.dump(yaml_content))
else:
with open(outfile, 'w') as f:
yaml.dump(yaml_content, f)
#!/usr/bin/env python
# coding: utf-8
#%%
import stac
#%%
s = stac.STAC('https://brazildatacube.dpi.inpe.br/stac/')
#%%
s.catalog
#%%
collection = s.collection('CB4_64_16D_STK-1')
collection
#%%
items = collection.get_items(
filter={
'bbox':
'-56.86523437500001,-15.919073517982413,-53.17382812500001,-13.902075852500483',
'time': '2016-09-13/2019-12-31'
})
items
