def collection2product_cli(collection, instrument, type, code, format, units, url, stac_version, outfile, ignore, pre_collection, verbose, access_token): constants = { 'instrument_type': instrument, 'metadata_type': type, 'platform_code': code, 'format_name': format, 'units': units, 'ignore': ignore, "is_pre_collection": pre_collection, 'verbose': verbose } _mapper = STAC_MAPPER_VERSIONS[stac_version] access_token = _check_token(access_token) c = stac.STAC(url, validate=False, access_token=access_token).collection(collection) yaml_content = stac2odc.collection.collection2product(c, _mapper(), **constants) if outfile is None: print(yaml.dump(yaml_content)) else: with open(outfile, 'w') as f: yaml.dump(yaml_content, f)
def test_items(self, stac_objects, requests_mock, runner): for k in stac_objects: s = stac.STAC(url, True) requests_mock.get(re.compile(url + '/stac'), json=stac_objects[k]['catalog.json'], status_code=200, headers={'content-type': 'application/json'}) requests_mock.get(re.compile(url + '/collections'), json=stac_objects[k]['collection.json'], status_code=200, headers={'content-type': 'application/json'}) requests_mock.get(re.compile(url + '/collections/my_collection1/items'), json=stac_objects[k]['items.json'], status_code=200, headers={'content-type': 'application/json'}) result = runner.invoke(stac.cli.items, [ '--url', url, '--collection-id', 'my_collection1', '--limit', 1, '--page', 1, '--datetime', '2016-05-03/2019-01-01', '--bbox', '-180,-90,180,90' ]) assert result.exit_code == 0 assert 'feature1' in result.output
def item2dataset_cli(collection, instrument, code, format, units, url, stac_version, basepath, outpath, ignore, max_items, pre_collection, verbose, download, download_out, advanced_filter, access_token): constants = { 'instrument_type': instrument, 'plataform_code': code, 'format_name': format, 'units': units, 'basepath': basepath, 'ignore': ignore, 'outpath': outpath, 'max_items': int(max_items), "is_pre_collection": pre_collection, 'verbose': verbose, "download": download, "download_out": download_out } mapper = STAC_MAPPER_VERSIONS[stac_version] _filter = {"collections": [collection]} if advanced_filter: _filter = { **_filter, **utils.prepare_advanced_filter(advanced_filter) } access_token = _check_token(access_token) s = stac.STAC(url, validate=False, access_token=access_token) stac2odc.item.item2dataset(s, _filter, mapper(), **constants)
def test_search(self, stac_objects, requests_mock, runner): for k in stac_objects: s = stac.STAC(url, True) requests_mock.get(re.compile(url + '/stac'), json=stac_objects[k]['catalog.json'], status_code=200, headers={'content-type': 'application/json'}) requests_mock.get(re.compile(url + '/collections'), json=stac_objects[k]['collection.json'], status_code=200, headers={'content-type': 'application/json'}) requests_mock.post(re.compile(url + '/stac/search'), json=stac_objects[k]['items.json'], status_code=200, headers={'content-type': 'application/json'}) intersects = json.dumps( stac_objects[k]['items.json']['features'][0]['geometry']) result = runner.invoke(stac.cli.search, [ '--url', url, '--collections', 'my_collection1', '--ids', 'feature1', '--intersects', intersects, '--limit', 1, '--next', 'aaa', '--page', 1, '--datetime', '2016-05-03/2019-01-01', '--bbox', '-180,-90,180,90' ]) assert result.exit_code == 0 assert 'feature1' in result.output
def describe(collection_name): """Get a description with available collection name from STAC. ## Parameters ### collection_name : string, required The collection name from list collections. ### token : string, required The BDC user token. ## Raise ### HTTPError If the STAC server is out of service. """ token = request.args.get('token') try: service = stac.STAC(config.STAC_URL, access_token=token) return jsonify(service.collections[collection_name]) except: return jsonify({'code': '403', 'message': 'Access error, forbidden!'})
def stac2json_cli(bdc_stac_search_bbox: str, bdc_stac_search_limit: int, bdc_stac_search_datetime: str, bdc_stac_url: str, bdc_stac_collection: str, bdc_access_token: str, output_file: str): # # client instance # client = stac.STAC(bdc_stac_url, access_token=bdc_access_token) # # get stac-features # collection = client.collection(bdc_stac_collection) # west, south, east, north items = collection.get_items( filter={ "bbox": bdc_stac_search_bbox, "datetime": bdc_stac_search_datetime, "limit": bdc_stac_search_limit }) # # export stac-features # with open(output_file, "w") as f: json.dump(items, f)
def test_collection(self, stac_objects, requests_mock): for k in stac_objects: s = stac.STAC(url, True) requests_mock.get(match_url, json=stac_objects[k]['catalog.json'], status_code=200, headers={'content-type': 'application/json'}) response = s.catalog requests_mock.get(match_url, json=stac_objects[k]['collection.json'], status_code=200, headers={'content-type': 'application/json'}) collection = s.collection('my_collection1') assert collection == stac_objects[k]['collection.json'] assert collection.keywords assert collection.version assert collection.license assert collection.properties assert collection.providers[0].name assert collection.providers[0].description assert collection.providers[0].roles assert collection.providers[0].url if k in ['0.8.0', '0.8.1']: assert collection.extent.spatial.bbox assert collection.extent.temporal.interval assert collection.summaries assert collection.summaries['val'].min == 0 assert collection.summaries['val'].max == 1 else: assert collection.extent.spatial assert collection.extent.temporal
def test_item(self, stac_objects, requests_mock): for k in stac_objects: s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True) if k == '0.9.0': requests_mock.get(re.compile(url + '/'), json=stac_objects[k]['catalog.json'], status_code=200, headers={'content-type': 'application/json'}) else: requests_mock.get(re.compile(url + '/stac'), json=stac_objects[k]['catalog.json'], status_code=200, headers={'content-type': 'application/json'}) requests_mock.get( re.compile(url + '/collections'), json=dict(collections=[stac_objects[k]['collection.json']]), status_code=200, headers={'content-type': 'application/json'}) catalog = s.catalog s.catalog #test for non empty catalog requests_mock.get(match_url, json=stac_objects[k]['collection.json'], status_code=200, headers={'content-type': 'application/json'}) collection = s.collection('my_collection1') requests_mock.get(match_url, json=stac_objects[k]['items.json'], status_code=200, headers={'content-type': 'application/json'}) itemcollection = s.collection('my_collection1').get_items() item = itemcollection.features[0] if collection.stac_version == '0.8.0': assert itemcollection.links assert item.properties.datetime assert item.properties.license assert item.properties.providers assert item.properties.title assert item.properties.created assert item.properties.updated assert itemcollection.type assert item.id == 'feature1' assert item.stac_version assert item.type assert item.bbox assert item.collection assert item.geometry.type assert item.geometry.coordinates assert item.properties assert item.links assert item.assets['thumbnail'].href assert item.assets['thumbnail'].title assert item.assets['thumbnail'].type
def test_search(self, stac_objects, requests_mock): for k in stac_objects: s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True) requests_mock.get(match_url, json=stac_objects[k]['catalog.json'], status_code=200, headers={'content-type': 'application/json'}) requests_mock.get( re.compile(url + '/collections'), json=dict(collections=[stac_objects[k]['collection.json']]), status_code=200, headers={'content-type': 'application/json'}) response = s.catalog requests_mock.get(match_url, json=stac_objects[k]['items.json'], status_code=200, headers={'content-type': 'application/json'}) response = s.search() assert response.features[0].id == 'feature1'
def test_catalog(self, stac_objects, requests_mock): for k in stac_objects: s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True) requests_mock.get(match_url, json=stac_objects[k]['catalog.json'], status_code=200, headers={'content-type': 'application/json'}) requests_mock.get( re.compile(url + '/collections'), json=dict(collections=[stac_objects[k]['collection.json']]), status_code=200, headers={'content-type': 'application/json'}) response = s.catalog assert s._catalog.stac_version assert s._catalog.id assert s._catalog.description assert s._catalog.title assert s._catalog.links[0].type assert s._catalog.links[0].title assert s._catalog.links[0].href assert s._catalog.links[0].rel assert response == ['my_collection1']
def item2dataset_cli(stac_collection, dc_product, url, outdir, max_items, engine_file, datacube_config, verbose, access_token, advanced_filter): _filter = {"collections": [stac_collection]} if advanced_filter: _filter = {**_filter, **prepare_advanced_filter(advanced_filter)} stac_service = stac.STAC(url, False, access_token=access_token) dc_index = datacube_index(datacube_config) features = create_feature_collection_from_stac_elements( stac_service, int(max_items), _filter) odc_datasets = stac2odc.item.item2dataset(engine_file, dc_product, features, dc_index, verbose=verbose) odc_datasets_definition_files = write_odc_element_in_yaml_file( odc_datasets, outdir) # add datasets definitions on datacube index # code adapted from: https://github.com/opendatacube/datacube-core/blob/develop/datacube/scripts/dataset.py ds_resolve = Doc2Dataset(dc_index, [dc_product]) doc_stream = remap_uri_from_doc( ui_path_doc_stream(odc_datasets_definition_files, uri=True)) datasets_on_stream = dataset_stream(doc_stream, ds_resolve) logger_message(f"Adding datasets", logger.info, True) for dataset in datasets_on_stream: try: dc_index.datasets.add(dataset, with_lineage=True) except (ValueError, MissingRecordError): logger_message(f"Error to add dataset ({dataset.local_uri})", logger.warning, True)
def test_collection_without_catalog(self, stac_objects, requests_mock): for k in stac_objects: s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True) requests_mock.get(match_url, json=stac_objects[k]['collection.json'], status_code=200, headers={'content-type':'application/json'}) response = s.collection('my_collection1') assert response == stac_objects[k]['collection.json']
def test_catalog(self, stac_objects, requests_mock, runner): for k in stac_objects: s = stac.STAC(url, True) requests_mock.get(match_url, json=stac_objects[k]['catalog.json'], status_code=200, headers={'content-type': 'application/json'}) result = runner.invoke(stac.cli.catalog, ['--url', url]) assert result.exit_code == 0 assert 'my_collection1' in result.output
def test_search(self, stac_objects, requests_mock): for k in stac_objects: s = stac.STAC(url, True) requests_mock.get(match_url, json=stac_objects[k]['items.json'], status_code=200, headers={'content-type': 'application/json'}) response = s.search() assert response.features[0].id == 'feature1'
def test_item_empty(self, stac_objects, requests_mock): for k in stac_objects: s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True) requests_mock.get(match_url, json=stac_objects[k]['collection.json'], status_code=200, headers={'content-type':'application/json'}) collection = s.collection('my_collection1') requests_mock.get(match_url, json=stac_objects[k]['items.json']['features'][0], status_code=200, headers={'content-type':'application/json'}) collection['links'].pop(1) assert collection.get_items() == stac.ItemCollection({})
def test_collection_missing(self, stac_objects, requests_mock): for k in stac_objects: s = stac.STAC(url, True) requests_mock.get(match_url, json=stac_objects[k]['catalog.json'], status_code=200, headers={'content-type': 'application/json'}) response = s.catalog requests_mock.get(match_url, exc=requests.exceptions.HTTPError) with pytest.raises(KeyError): s.collection('missing_collection')
def search(): """Get a description with available collection name from STAC. ## Parameters ### token : string, required The BDC user token. ## JSON Object ### Test data for POST: { "collections": ["CB4_64_16D_STK-1"], "bbox": [-46.01348876953125, -23.08478515994374, -45.703125, -23.34856015148709], "interval": ["2018-08-01","2019-07-31"], "limit": 10 } ## Raise ### HTTPError If the STAC server is out of service. """ token = request.args.get('token') if request.method == 'POST': try: collections = request.json['collections'] bbox = tuple(request.json['bbox']) start_date = request.json['interval'][0] end_date = request.json['interval'][1] limit = request.json['limit'] try: query = { 'collections': collections, 'bbox': bbox, 'datetime': f'{start_date}/{end_date}', 'limit': limit } items = stac.STAC(config.STAC_URL, access_token=token).search(query) return jsonify({"query": query, "result": items.features}) except: return jsonify({ 'code': '403', 'message': 'Access error, forbidden!' }) except: return jsonify({'code': '400', 'message': 'Request JSON error!'})
def __init__(self): """Build DataCube object with config parameters including access token, STAC url and earth observation service url.""" if len(config.ACCESS_TOKEN) == 0: config.ACCESS_TOKEN = input("Please insert a valid user token from BDC Auth: ") if len(config.EOCUBE_URL) == 0: config.EOCUBE_URL = input("Please insert a valid url for EO Service: ") if len(config.STAC_URL) == 0: config.STAC_URL = input("Please insert a valid url for STAC Service: ") self.stac_service = stac.STAC( config.STAC_URL, access_token=config.ACCESS_TOKEN ) self.query = None self.items = None self.images = []
def test_stac_read_forbidden(self, stac_objects, requests_mock): for k in stac_objects: s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True) requests_mock.get(match_url, json=stac_objects[k]['catalog.json'], status_code=200, headers={'content-type': 'application/json'}) requests_mock.get( re.compile(url + '/collections'), json=dict(collections=[stac_objects[k]['collection.json']]), status_code=200, headers={'content-type': 'application/json'}) response = s.catalog requests_mock.get(match_url, json=stac_objects[k]['items.json'], status_code=200, headers={'content-type': 'application/json'}) response = s.search() item = response.features[0] requests_mock.head(item.assets['thumbnail']['href'], status_code=403, headers={'content-type': 'application/json'}) with pytest.raises(requests.exceptions.HTTPError) as err: _ = item.read('thumbnail') assert err.value.response.status_code == 403 # No reason is set when using Mock assert str(err.value) == '(None) Missing Authentication Token.' item.assets['thumbnail']['href'] += '?access_token=test' requests_mock.head(item.assets['thumbnail']['href'], status_code=403, headers={'content-type': 'application/json'}) with pytest.raises(requests.exceptions.HTTPError) as err: _ = item.read('thumbnail') assert err.value.response.status_code == 403 assert str( err.value ) == "(None) You don't have permission to request this resource."
def test_collection(self, stac_objects, requests_mock, runner): for k in stac_objects: s = stac.STAC(url, True) requests_mock.get(re.compile(url + '/stac'), json=stac_objects[k]['catalog.json'], status_code=200, headers={'content-type': 'application/json'}) requests_mock.get(re.compile(url + '/collections'), json=stac_objects[k]['collection.json'], status_code=200, headers={'content-type': 'application/json'}) result = runner.invoke( stac.cli.collection, ['--url', url, '--collection-id', 'my_collection1']) assert result.exit_code == 0 assert 'my_collection1' in result.output
def item2dataset_cli(collection, instrument, code, format, units, url, basepath, outpath, ignore, max_items, pre_collection, verbose, download, download_out): constants = { 'instrument_type': instrument, 'plataform_code': code, 'format_name': format, 'units': units, 'basepath': basepath, 'ignore': ignore, 'outpath': outpath, 'max_items': int(max_items), "is_pre_collection": pre_collection, 'verbose': verbose, "download": download, "download_out": download_out } s = stac.STAC(url, True) c = s.collection(collection) stac2odc.item.item2dataset(c, mapper.Stac2ODCMapper08(), **constants)
def test_item_id(self, stac_objects, requests_mock): for k in stac_objects: s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True) requests_mock.get(match_url, json=stac_objects[k]['catalog.json'], status_code=200, headers={'content-type':'application/json'}) response = s.catalog requests_mock.get(match_url, json=stac_objects[k]['collection.json'], status_code=200, headers={'content-type':'application/json'}) collection = s.collection('my_collection1') requests_mock.get(match_url, json=stac_objects[k]['items.json']['features'][0], status_code=200, headers={'content-type':'application/json'}) response = collection.get_items(item_id='feature1') assert response.id == 'feature1'
def test_empty_bbox(self, stac_objects, requests_mock): for k in stac_objects: s = stac.STAC(url + "/stac" if k != '0.9.0' else url, True) requests_mock.get(match_url, json=stac_objects[k]['catalog.json'], status_code=200, headers={'content-type': 'application/json'}) requests_mock.get( re.compile(url + '/collections'), json=dict(collections=[stac_objects[k]['collection.json']]), status_code=200, headers={'content-type': 'application/json'}) with pytest.raises(TypeError): s.search(bbox="") with pytest.raises(TypeError): s.search(bbox=-90)
def collection2product_cli(collection, instrument, type, code, format, units, url, outfile, ignore, pre_collection, verbose): constants = { 'instrument_type': instrument, 'metadata_type': type, 'platform_code': code, 'format_name': format, 'units': units, 'ignore': ignore, "is_pre_collection": pre_collection, 'verbose': verbose } s = stac.STAC(url, True) c = s.collection(collection) yaml_content = stac2odc.collection.collection2product(c, mapper.Stac2ODCMapper08(), **constants) if outfile is None: print(yaml.dump(yaml_content)) else: with open(outfile, 'w') as f: yaml.dump(yaml_content, f)
def collections(): """Get a list with available collections from STAC. ## Parameters ### token : string, optional The BDC user token. ## Raise ### HTTPError If the STAC server is out of service. """ token = request.args.get('token') try: service = stac.STAC(config.STAC_URL, access_token=token) return jsonify({'collections': list(service.collections.keys())}) except: return jsonify({'code': '403', 'message': 'Access error, forbidden!'})
def collection2product_cli(collection: str, url: str, outdir: str, engine_file: str, datacube_config: str, access_token, verbose: bool): collection_definition = stac.STAC( url, False, access_token=access_token).collection(collection) odc_element = stac2odc.collection.collection2product(engine_file, collection_definition, verbose=verbose) product_definition_file = write_odc_element_in_yaml_file( odc_element, os.path.join(outdir, f'{collection}.yaml')) # code adapted from: https://github.com/opendatacube/datacube-core/blob/develop/datacube/scripts/product.py for path_descriptor, parsed_doc in read_documents( *[product_definition_file]): try: dc_index = datacube_index(datacube_config) _type = dc_index.products.from_doc(parsed_doc) logger_message(f'Adding {_type.name}', logger.info, verbose) dc_index.products.add(_type) except InvalidDocException as e: logger_message(f'Error to add product: {str(e)}', logger.warning, True)
if __name__ == '__main__': import stac import yaml import stac2odc.item import stac2odc.collection from stac2odc.mapper import Stac2ODCMapper09 constants = { 'instrument_type': 'AWFI', 'metadata_type': 'eo', 'platform_code': 'CBERS04', 'format_name': 'GeoTiff', 'units': 'm', 'ignore': ['quality'], "is_pre_collection": False, 'verbose': True } outfile = 'test.yaml' s = stac.STAC('http://brazildatacube.dpi.inpe.br/stac/', False) c = s.collection('CB4_64_16D_STK-1') yaml_content = stac2odc.collection.collection2product( c, Stac2ODCMapper09(), **constants) if outfile is None: print(yaml.dump(yaml_content)) else: with open(outfile, 'w') as f: yaml.dump(yaml_content, f)
def test_stac(self): s = stac.STAC(url, True) assert s.url == url assert repr(s) == f'stac("{url}")' assert str(s) == f'<STAC [{url}]>'
if __name__ == '__main__': import stac import yaml import stac2odc.item import stac2odc.collection from stac2odc.mapper import Stac2ODCMapper08 constants = { 'instrument_type': 'AWFI', 'metadata_type': 'eo', 'platform_code': 'CBERS04', 'format_name': 'GeoTiff', 'units': 'm', 'ignore': ['quality'], "is_pre_collection": False, 'verbose': True } outfile = 'test.yaml' s = stac.STAC('http://brazildatacube.dpi.inpe.br/bdc-stac/0.8.0/', True) c = s.collection('CB4_64_16D_STK_v1') yaml_content = stac2odc.collection.collection2product( c, Stac2ODCMapper08(), **constants) if outfile is None: print(yaml.dump(yaml_content)) else: with open(outfile, 'w') as f: yaml.dump(yaml_content, f)
#!/usr/bin/env python # coding: utf-8 #%% import stac #%% s = stac.STAC('https://brazildatacube.dpi.inpe.br/stac/') #%% s.catalog #%% collection = s.collection('CB4_64_16D_STK-1') collection #%% items = collection.get_items( filter={ 'bbox': '-56.86523437500001,-15.919073517982413,-53.17382812500001,-13.902075852500483', 'time': '2016-09-13/2019-12-31' }) items