def index_data(bbox): start_date = '2020-05-01' end_date = '2021-01-06' print(bbox) collections = ['sentinel-s2-l2a-cogs'] config = { 'collections': collections, 'bbox': bbox, 'datetime': f"{start_date}/{end_date}" } STAC_API_URL = 'https://explorer.sandbox.dea.ga.gov.au/stac/' os.environ['STAC_API_URL'] = STAC_API_URL srch = Search().search(**config) found_items = srch.found() print(f"Found {found_items} items that can be indexed") dc = datacube.Datacube() indexed, failed = stac_api_to_odc(dc, 's2_l2a', None, False, False, config) print(f"Indexed {indexed} out of {found_items} with {failed} failures.")
def run(params, url, sleep=None): search = Search(api_url=url, **params) logger.debug(f"Searching {url}") found = search.found() logger.debug(f"Total items found: {found}") if found < MAX_ITEMS_REQUEST: logger.info(f"Making single request for {found} items") items = search.items() for i, item in enumerate(items): resp = SNS_CLIENT.publish(TopicArn=SNS_TOPIC, Message=json.dumps(item._data)) if (i % 500) == 0: logger.debug(f"Added {i+1} items to Cirrus") #if resp['StatusCode'] != 200: # raise Exception("Unable to publish") if sleep: time.sleep(sleep) logger.debug(f"Published {len(items)} items to {SNS_TOPIC}") else: # bisection nbatches = 2 logger.info( f"Too many Items for single request, splitting into {nbatches} batches by date range" ) for params in split_request(params, nbatches): run(params, url)
def run(self): if not self.isRunning: self.isRunning = True items_count = 0 self.search_is_started.emit( "[" + str(datetime.datetime.now().strftime("%H:%M:%S")) + "]" + " Ищу...") # is really shit-shit code, # но ребятишки, сделавшие API, видимо не подумали о том, # что может потребоваться передавать параметры через OR (||) if isinstance(self.query, dict): if self.intersects is not None: search = Search(intersects=self.intersects, time=self.time, query=self.query) else: search = Search(time=self.time, query=self.query) items_count = search.found() self.files_are_found.emit(items_count, self.query["collection"]["eq"]) elif isinstance(self.query, list): for single_query in self.query: if self.intersects is not None: search = Search(intersects=self.intersects, time=self.time, query=single_query) else: search = Search(time=self.time, query=single_query) items_count += search.found() self.files_are_found.emit(items_count, self.query[0]["collection"]["eq"])
def search_landsat_data(coordinates, cloud_cover_lt): search = Search( bbox=get_tiny_bbox(coordinates), query={ "eo:cloud_cover": { "lt": cloud_cover_lt }, "collection": { "eq": "landsat-8-l1" }, }, sort=[{ "field": "eo:cloud_cover", "direction": "asc" }], ) # improvement: filter by date, cloud cover here search_items = search.items() if not len(search_items): exit_cli( print, "No data matched your search, please try different parameters.") landsat_item = search_items[0] return landsat_item
def main(scenes=None, print_md=None, print_cal=False, save=None, download=None, **kwargs): """ Main function for performing a search """ if scenes is None: # get scenes from search search = Search(**kwargs) scenes = Scenes(search.scenes(), properties=kwargs) else: scenes = Scenes.load(scenes) # print metadata if print_md is not None: scenes.print_scenes(print_md) # print calendar if print_cal: print(scenes.text_calendar()) # save all metadata in JSON file if save is not None: scenes.save(filename=save) print('%s scenes found' % len(scenes)) # download files given keys if download is not None: for key in download: scenes.download(key=key) return scenes
def lambda_handler(event, context={}): logger.debug('Event: %s' % json.dumps(event)) url = event.get('url') params = event.get('search', {}) max_items_batch = event.get('max_items_batch', 15000) sleep = event.get('sleep', None) process = event.get('process', None) # search API search = Search(url=url, **params) logger.debug(f"Searching {url}") found = search.found() logger.debug(f"Total items found: {found}") if found <= MAX_ITEMS_REQUEST: return run(params, url, sleep=sleep, process=process) elif hasattr(context, "invoked_function_arn"): nbatches = int(found / max_items_batch) + 1 if nbatches == 1: submit_batch_job(event, context.invoked_function_arn, definition='lambda-as-batch') else: for request in split_request(params, nbatches): event['search'] = request submit_batch_job(event, context.invoked_function_arn, definition='lambda-as-batch') logger.info(f"Submitted {nbatches} batches") return else: run(params, url, sleep=sleep, process=process)
def search_satellite_data(coordinates, cloud_cover_lt, product="landsat"): """ coordinates: bounding box's coordinates cloud_cover_lt: maximum cloud cover product: landsat, sentinel """ if product == "landsat": product = "landsat-8-l1" elif product == "sentinel": product = "sentinel-2-l1c" search = Search( bbox=get_tiny_bbox(coordinates), query={ "eo:cloud_cover": {"lt": cloud_cover_lt}, "collection": {"eq": product}, }, sort=[{"field": "eo:cloud_cover", "direction": "asc"}], ) # improvement: filter by date, cloud cover here search_items = search.items() if not len(search_items): exit_cli(print, "No data matched your search, please try different parameters.") # return the first result item = search_items[0] return item
def main(review=False, printsearch=False, printmd=None, printcal=False, load=None, save=None, append=False, download=None, source='aws_s3', **kwargs): """ Main function for performing a search """ if load is None: if printsearch: txt = 'Search for scenes matching criteria:\n' for kw in kwargs: if kw == 'intersects': geom = json.dumps(json.loads(kwargs[kw])['geometry']) txt += ('{:>20}: {:<40} ...\n'.format(kw, geom[0:70])) else: txt += ('{:>20}: {:<40}\n'.format(kw, kwargs[kw])) print(txt) # get scenes from search search = Search(**kwargs) scenes = Scenes(search.scenes()) else: scenes = Scenes.load(load) if review: if not os.getenv('IMGCAT', None): raise ValueError( 'Set IMGCAT envvar to terminal image display program to use review feature' ) scenes.review_thumbnails() # print summary if printmd is not None: scenes.print_scenes(printmd) # print calendar if printcal: print(scenes.text_calendar()) # save all metadata in JSON file if save is not None: scenes.save(filename=save, append=append) print('%s scenes found' % len(scenes)) # download files given keys if download is not None: for key in download: scenes.download(key=key, source=source) return scenes
def get_items(srch: Search, limit: bool) -> Generator[Tuple[dict, str, bool], None, None]: if limit: items = srch.items(limit=limit) else: items = srch.items() for metadata in items.geojson()["features"]: uri, relative = guess_location(metadata) yield (metadata, uri, relative)
def search(geometry: dict, datetime: date) -> object: logger.info('Searching images') kwargs = { 'intersects': geometry, 'datetime': datetime.isoformat(), 'collections': [LANDSAT] } search = Search(url=URL, **kwargs) items = search.items() logger.debug('Found: %d', len(items)) return next(iter(items))
def search_scenes(bbox, collection='landsat-8-l1', cloud_cover=(0, 10)): search = Search(bbox=bbox, query={ 'eo:cloud_cover': { 'gt': cloud_cover[0], 'lt': cloud_cover[1] }, 'collection': { 'eq': collection } }) return search.items()
def stac_search(extent, start_date, end_date): """ Convert lat, lon to pathrows """ logger.info("Querying STAC for area: {} and times: {} - {} (UTC)".format( extent, start_date, end_date)) srch = Search(bbox=extent, time='{}T00:00:00Z/{}T23:59:59Z'.format( start_date, end_date), url="https://sat-api.developmentseed.org/stac/search") try: logger.info("Found {} items".format(srch.found())) return srch except KeyError: return None
def get_items( srch: Search, limit: Optional[int] ) -> Generator[Tuple[dict, str, bool], None, None]: if limit: items = srch.items(limit=limit) else: items = srch.items() # Workaround bug in STAC Search that doesn't stop at the limit for count, metadata in enumerate(items.geojson()["features"]): # Stop at the limit if it's set if (limit is not None) and (count > limit): break uri, relative = guess_location(metadata) yield (metadata, uri, relative)
def buscarResultadosSatelite(minLon, minLat, maxLon, maxLat, minDate, maxDate, maxCloudCover, satellite): search = Search.search(bbox=[minLon, minLat, maxLon, maxLat], datetime=minDate + '/' + maxDate, property=["eo:cloud_cover<" + str(maxCloudCover)], collection=satellite) items = search.items() return items
def lambda_handler(event, context={}): logger.debug('Event: %s' % json.dumps(event)) # if this is batch, output to stdout if not hasattr(context, "invoked_function_arn"): logger.addHandler(logging.StreamHandler()) # parse input #s3urls = event['s3urls'] #suffix = event.get('suffix', 'json') #credentials = event.get('credentials', {}) #requester_pays = credentials.pop('requester_pays', False) ###### url = event.get('url') params = event.get('search', {}) max_items_batch = event.get('max_items_batch', 15000) sleep = event.get('sleep', None) # search API search = Search(api_url=url, **params) logger.debug(f"Searching {url}") found = search.found() logger.debug(f"Total items found: {found}") if found <= MAX_ITEMS_REQUEST: return run(params, url, sleep=sleep) elif hasattr(context, "invoked_function_arn"): nbatches = int(found / max_items_batch) + 1 if nbatches == 1: submit_batch_job(event, context.invoked_function_arn, definition='lambda-as-batch') else: for request in split_request(params, nbatches): event['search'] = request submit_batch_job(event, context.invoked_function_arn, definition='lambda-as-batch') logger.info(f"Submitted {nbatches} batches") return else: run(params, url, sleep=sleep)
def retrieveSatelliteImage(min_lon, min_lat, max_lon, max_lat): '''retrieve a satellite image with min and max coordinates as corners''' start_date = '2021-03-25T00:00:00Z' end_date = date.today().strftime('%Y-%m-%dT00:00:00Z') search = Search(bbox=[min_lon, min_lat, max_lon, max_lat], datetime=start_date + '/' + end_date, url='https://earth-search.aws.element84.com/v0') print(search) items = search.items(limit=1) keys = [k for i in items for k in i.assets] filename = items[0].download(keys[0], filename_template='satellite_images/image') return filename
def search_image(date, bb, prop): """Searches Satellite-Image for given Boundingbox, Date and Properties :parameter: singel Date, Bounding Box as List Properties as String :return: statsac.Item Object with the lowest cloud-coverage for the given Date""" image = [] # search image for given date or periode of time, # always takes the first image search = Search(bbox=bb, datetime=date, property=[prop], sort=[{ 'field': 'eo:cloud_cover', 'direction': 'asc' }]) items = search.items() # filter for Landsatimages since Sentinel doesn't work and the # collection option for sat-search seems to be broken workaround = [str(item) for item in items] counter = 0 for z in workaround: if 'S2' in z: counter += 1 pass else: image.append(items[counter]) break # check if image was found assert len(image) == 1, 'No Images for given Parameters found. ' \ 'Please try new ones' return items[counter]
def main(items=None, printmd=None, printcal=None, printassets=None, found=False, filename_template='${collection}/${date}/${id}', save=None, download=None, requester_pays=False, headers=None, **kwargs): """ Main function for performing a search """ if items is None: ## if there are no items then perform a search search = Search.search(headers=headers, **kwargs) if found: num = search.found(headers=headers) print('%s items found' % num) return num items = search.items(headers=headers) else: # otherwise, load a search from a file items = ItemCollection.open(items) print('%s items found' % len(items)) # print metadata if printmd is not None: print(items.summary(printmd)) # print calendar if printcal: print(items.calendar(printcal)) if printassets: print(items.assets_definition()) # save all metadata in JSON file if save is not None: items.save(filename=save) # download files given `download` keys if download is not None: if 'ALL' in download: # get complete set of assets download = set([k for i in items for k in i.assets]) for key in download: items.download(key=key, filename_template=filename_template, requester_pays=requester_pays) return items
def download_images(): # Bounding box (min lon, min lat, max lon, max lat) search = Search.search( bbox=[-123.268003, 49.170589, -122.970262, 49.326161], datetime="2019-01-01/2019-06-30", property=["eo:cloud_cover<20"], collection="landsat-8-l1") items = search.items(limit=10) matadata = items.save('sat-search-images_data.json') print("Downloaded satellite images information:") print(items.summary(['date', 'id', 'eo:cloud_cover'])) filenames = items.download("thumbnail")
def post(self): body = request.get_json() geom = { "type": "Polygon", "coordinates": [[[-66.3958740234375, 43.305193797650546], [-64.390869140625, 43.305193797650546], [-64.390869140625, 44.22945656830167], [-66.3958740234375, 44.22945656830167], [-66.3958740234375, 43.305193797650546]]] } print(body) cloud = "eo:cloud_cover<" + "5" #body['cloud'] time = "2018-02-01/2018-02-10" #body["time"] search = Search(intersects=geom, time=time, property=[cloud]) items = search.items() print(search.found()) for item in items: print(item.assets["thumbnail"]["href"]) return 200
def get_satellite_image(coords): """ Generate an image given min and max gps coordinate pairs """ # set the date parameters for searching start_date = '2021-04-07T00:00:00Z' end_date = date.today().strftime('%Y-%m-%dT00:00:00Z') # search for a satellite image match search = Search(bbox=coords, datetime=start_date + '/' + end_date, url='https://earth-search.aws.element84.com/v0') # only record the latest one items = search.items(limit=1) # try to find a better way to do this keys = [k for i in items for k in i.assets] # download the latest one filename = items[0].download(keys[0], filename_template='satellite_images/image') return filename
def search_image(date, bb, prop): """Searches Satellite-Image for given Boundingbox, Date and Properties :parameter: singel Date or range of Days, Bounding Box as List and Properties as String :return: List containing statsac.Item Object with the lowest cloud-coverage for the given Date""" image = [] # search image for given date or periode of time, always takes the first image search = Search(bbox=bb, datetime=date, property=[prop], sort=[{ 'field': 'eo:cloud_cover', 'direction': 'desc' }]) items = search.items() image.append(items[-1]) assert len( image ) == 1, 'No Images for given Parameters found. Please try new ones' return image
def stac_api_to_odc( dc: Datacube, products: list, limit: int, update: bool, allow_unsafe: bool, config: dict, **kwargs, ) -> Tuple[int, int]: # QA the BBOX if config["bbox"]: assert ( len(config["bbox"]) == 4 ), "Bounding box must be of the form lon-min,lat-min,lon-max,lat-max" # QA the search srch = Search().search(**config) n_items = srch.found() logging.info("Found {} items to index".format(n_items)) if n_items > 10000: logging.warning( "More than 10,000 items were returned by your query, which is greater than the API limit" ) if n_items == 0: logging.warning("Didn't find any items, finishing.") return 0, 0 # Get a generator of (stac, uri, relative_uri) tuples potential_items = get_items(srch, limit) # Get a generator of (dataset, uri) doc2ds = Doc2Dataset(dc.index, **kwargs) datasets = transform_items(doc2ds, potential_items) # Do the indexing of all the things return index_update_datasets(dc, datasets, update, allow_unsafe)
def main(items=None, printmd=None, printcal=False, found=False, save=None, download=None, requestor_pays=False, **kwargs): """ Main function for performing a search """ if items is None: ## if there are no items then perform a search search = Search.search(**kwargs) if found: num = search.found() print('%s items found' % num) return num items = search.items() else: # otherwise, load a search from a file items = ItemCollection.load(items) print('%s items found' % len(items)) # print metadata if printmd is not None: print(items.summary(printmd)) # print calendar if printcal: print(items.calendar()) # save all metadata in JSON file if save is not None: items.save(filename=save) # download files given `download` keys if download is not None: if 'ALL' in download: # get complete set of assets download = set([k for i in items for k in i.assets]) for key in download: items.download(key=key, path=config.DATADIR, filename=config.FILENAME, requestor_pays=requestor_pays) return items
def main(items=None, printmd=None, printcal=False, found=False, save=None, download=None, requestor_pays=False, **kwargs): """ Main function for performing a search """ if items is None: ## if there are no items then perform a search search = Search.search(**kwargs) if found: num = search.found() print('%s items found' % num) return num items = search.items() else: # otherwise, load a search from a file items = Items.load(items) print('%s items found' % len(items)) # print metadata if printmd is not None: print(items.summary(printmd)) # print calendar if printcal: print(items.calendar()) # save all metadata in JSON file if save is not None: items.save(filename=save) # download files given `download` keys if download is not None: if 'ALL' in download: # get complete set of assets download = set([k for i in items for k in i.assets]) for key in download: items.download(key=key, path=config.DATADIR, filename=config.FILENAME, requestor_pays=requestor_pays) return items
if OSM: OSM_DATA = CONFIGS["osm_query"] except KeyError as error: print("Invalid Config file, propaly not all ") print(error) sys.exit() assert os.path.exists(JSON_PATH), "Json file does not exist" try: FEATURES = load_geojson_features_to_dict(JSON_PATH) if OSM: FEATURES = grab_osm_data(OSM_DATA, FEATURES) BBOX = get_bbox(FEATURES) GEOM_LIST = get_geom_list(FEATURES, BUFFER_DISTANCE) SEARCH = Search.search(bbox=BBOX, datetime=DATETIME_FRAME, collection=COLLECTION, property=PROPERTY) ITEMS = SEARCH.items() print("Found {} tiles matching".format(SEARCH.found())) DATE_LIST = [] NDVI_LIST = [] for i in range(len(FEATURES)): NDVI_LIST.append([]) DATE_LIST.append([]) for item_counter, item in enumerate(ITEMS): print("Processing Tile {} of {}".format(item_counter + 1, len(ITEMS))) URL_PAIR = get_url_pair(item, COLLECTION)
import numpy as np from pathlib import Path # Must be installed via pip; conda version outdated. from satsearch import Search # Specify geometry geom = { "type": "Polygon", "coordinates": [[[-66.3958740234375, 43.305193797650546], [-64.390869140625, 43.305193797650546], [-64.390869140625, 44.22945656830167], [-66.3958740234375, 44.22945656830167], [-66.3958740234375, 43.305193797650546]]] } # Search for all landsat-8-l1 data intersecting with that geometry search = Search.search(intersects=geom, collection='landsat-8-l1') # Collect urls/items having to do with that data items = search.items() # print summary print(items.summary()) # Get just one item and download all associated assets (TIFs for bands B0-B11): # Each item with all bands is roughly 1/2 a gb? items = search.items(limit=1) print(items.summary()) items.download_assets(path=str(Path.cwd()) + '/${date}')
def get_files(self, instance): bands = instance.bands # geocoding # place = self.context['location'] # response = requests.get(f"https://nominatim.openstreetmap.org/search?q={place}&format=json&polygon_geojson=1&addressdetails=0&limit=1") # json_object = response.json() # bb = json_object[0]['boundingbox'] # bbx = [float(x) for x in bb] # aws configuration url = 'https://earth-search.aws.element84.com/v0' # URL to Sentinel 2 AWS catalog collection = 'sentinel-s2-l2a-cogs' # aws search parameter startDate = self.context['date_from'] endDate = self.context['date_to'] # location = [ # 11.40020, # 53.63612, # 11.44569, # 53.62385 # ] bbox_search = Search( bbox=self.context['location'], datetime=startDate + "/" + endDate, query={ 'eo:cloud_cover': { 'lt': 50 }, 'sentinel:data_coverage': { 'gt': 80 } }, collections=[collection], url=url, sortby='+properties.eo:cloud_cover', ) items = bbox_search.items() downloads = {} for i, item in enumerate(items): data = {} data['Product ID'] = item.properties["sentinel:product_id"] data['Preview'] = item.asset("thumbnail")["href"] data['Date'] = item.properties["datetime"] data['Cloud cover'] = item.properties["eo:cloud_cover"] data['Latitude band'] = item.properties["sentinel:latitude_band"] data['Grid'] = item.properties["sentinel:grid_square"] data['UTM Zone'] = item.properties["sentinel:utm_zone"] data['Sequence'] = item.properties["sentinel:sequence"] data['Projection'] = item.properties["proj:epsg"] data['Data coverage'] = item.properties["sentinel:data_coverage"] for band in bands.split(','): data[band] = item.asset(band)["href"] downloads[i] = data df = pd.DataFrame.from_dict(downloads, orient='index') df = df.drop_duplicates('Grid') results = df.to_dict(orient='index') return results
from satsearch import Search import sys import numpy as np import rasterio as rio from itertools import product from rasterio import windows import matplotlib.pyplot as plt import multiprocessing as mp print("Number of processors: ", mp.cpu_count()) search = Search(bbox=[8.66744, 49.41217, 8.68465, 49.42278], datetime='2018-06-01/2018-08-04', property=["eo:cloud_cover<5"]) items = search.items() item = items[6] def search_image(date, bb, prop): """Searches Satellite-Image for given Boundingbox, Date and Properties :parameter: singel Date or range of Days, Bounding Box as List and Properties as String :return: List containing statsac.Item Object with the lowest cloud-coverage for the given Date""" image = [] # search image for given date or periode of time, always takes the first image search = Search(bbox=bb, datetime=date, property=[prop], sort=[{
def ls_make_hits_dataframe(aoi_geojson_file, time): pids, bands, paths, rows, dates, reds = [], [], [], [], [], [] greens, blues = [], [] nirs, pixel_qas = [], [] print("return_stac_hits") geom = return_geo_query_body(aoi_geojson_file, time = time) date_range = _return_date_query(time) search = Search.search(intersects=geom, datetime=date_range, collection='landsat-8-l1') print('intersects search: %s items' % search.found()) items = search.items() print('%s items' % len(items)) print('%s collections' % len(items._collections)) print(items._collections) for item in items: #print(item) product_id = item.id path = product_id[3:6] row = product_id[6:9] #print ("PATH ROW", path, row) bandd = _return_band_dict(item) #print(bandd['red']) file_part = bandd['red'].split('/')[-1] print(file_part) product = file_part[0:9] band = file_part.split('_')[-1] date = str(item.date) red = bandd['red'] green = bandd['green'] blue = bandd['blue'] nir = bandd['nir'] pixel_qa = bandd['BQA'] pids.append(product) bands.append(band) paths.append(path) rows.append(row) dates.append(date) reds.append(red) greens.append(green) blues.append(blue) nirs.append(nir) pixel_qas.append(pixel_qa) panda_input_dict = dict(product=pids, band=bands, path=paths, row=rows, date=dates, red=reds, green=greens, blue=blues, nir=nirs, pixel_qa=pixel_qas) DF = pd.DataFrame(panda_input_dict, columns=['product', 'band', 'path', 'row', 'date', 'red', 'green', 'blue', 'nir', 'pixel_qa']) cols = DF.columns.tolist() return DF
from satsearch import Search from satstac import Item search = Search(bbox=[-110, 39.5, -105, 40.5]) print('bbox search: %s items' % search.found()) search = Search(time='2018-02-12T00:00:00Z/2018-03-18T12:31:12Z') print('time search: %s items' % search.found()) search = Search(query={'eo:cloud_cover': {'lt': 10}}) print('cloud_cover search: %s items' % search.found()) search = Search(bbox=[-110, 39.5, -105, 40.5], time='2018-02-12T00:00:00Z/2018-03-18T12:31:12Z', query={'eo:cloud_cover': { 'lt': 10 }}) print('%s items' % search.found()) geom = { "type": "Polygon", "coordinates": [[[-66.3958740234375, 43.305193797650546], [-64.390869140625, 43.305193797650546], [-64.390869140625, 44.22945656830167], [-66.3958740234375, 44.22945656830167], [-66.3958740234375, 43.305193797650546]]] } search = Search(intersects=geom) print('intersects search: %s items' % search.found())