def run(self): if not self.isRunning: self.isRunning = True items_count = 0 self.search_is_started.emit( "[" + str(datetime.datetime.now().strftime("%H:%M:%S")) + "]" + " Ищу...") # is really shit-shit code, # но ребятишки, сделавшие API, видимо не подумали о том, # что может потребоваться передавать параметры через OR (||) if isinstance(self.query, dict): if self.intersects is not None: search = Search(intersects=self.intersects, time=self.time, query=self.query) else: search = Search(time=self.time, query=self.query) items_count = search.found() self.files_are_found.emit(items_count, self.query["collection"]["eq"]) elif isinstance(self.query, list): for single_query in self.query: if self.intersects is not None: search = Search(intersects=self.intersects, time=self.time, query=single_query) else: search = Search(time=self.time, query=single_query) items_count += search.found() self.files_are_found.emit(items_count, self.query[0]["collection"]["eq"])
def run(params, url, sleep=None): search = Search(api_url=url, **params) logger.debug(f"Searching {url}") found = search.found() logger.debug(f"Total items found: {found}") if found < MAX_ITEMS_REQUEST: logger.info(f"Making single request for {found} items") items = search.items() for i, item in enumerate(items): resp = SNS_CLIENT.publish(TopicArn=SNS_TOPIC, Message=json.dumps(item._data)) if (i % 500) == 0: logger.debug(f"Added {i+1} items to Cirrus") #if resp['StatusCode'] != 200: # raise Exception("Unable to publish") if sleep: time.sleep(sleep) logger.debug(f"Published {len(items)} items to {SNS_TOPIC}") else: # bisection nbatches = 2 logger.info( f"Too many Items for single request, splitting into {nbatches} batches by date range" ) for params in split_request(params, nbatches): run(params, url)
def index_data(bbox): start_date = '2020-05-01' end_date = '2021-01-06' print(bbox) collections = ['sentinel-s2-l2a-cogs'] config = { 'collections': collections, 'bbox': bbox, 'datetime': f"{start_date}/{end_date}" } STAC_API_URL = 'https://explorer.sandbox.dea.ga.gov.au/stac/' os.environ['STAC_API_URL'] = STAC_API_URL srch = Search().search(**config) found_items = srch.found() print(f"Found {found_items} items that can be indexed") dc = datacube.Datacube() indexed, failed = stac_api_to_odc(dc, 's2_l2a', None, False, False, config) print(f"Indexed {indexed} out of {found_items} with {failed} failures.")
def lambda_handler(event, context={}): logger.debug('Event: %s' % json.dumps(event)) url = event.get('url') params = event.get('search', {}) max_items_batch = event.get('max_items_batch', 15000) sleep = event.get('sleep', None) process = event.get('process', None) # search API search = Search(url=url, **params) logger.debug(f"Searching {url}") found = search.found() logger.debug(f"Total items found: {found}") if found <= MAX_ITEMS_REQUEST: return run(params, url, sleep=sleep, process=process) elif hasattr(context, "invoked_function_arn"): nbatches = int(found / max_items_batch) + 1 if nbatches == 1: submit_batch_job(event, context.invoked_function_arn, definition='lambda-as-batch') else: for request in split_request(params, nbatches): event['search'] = request submit_batch_job(event, context.invoked_function_arn, definition='lambda-as-batch') logger.info(f"Submitted {nbatches} batches") return else: run(params, url, sleep=sleep, process=process)
def stac_search(extent, start_date, end_date): """ Convert lat, lon to pathrows """ logger.info("Querying STAC for area: {} and times: {} - {} (UTC)".format( extent, start_date, end_date)) srch = Search(bbox=extent, time='{}T00:00:00Z/{}T23:59:59Z'.format( start_date, end_date), url="https://sat-api.developmentseed.org/stac/search") try: logger.info("Found {} items".format(srch.found())) return srch except KeyError: return None
def lambda_handler(event, context={}): logger.debug('Event: %s' % json.dumps(event)) # if this is batch, output to stdout if not hasattr(context, "invoked_function_arn"): logger.addHandler(logging.StreamHandler()) # parse input #s3urls = event['s3urls'] #suffix = event.get('suffix', 'json') #credentials = event.get('credentials', {}) #requester_pays = credentials.pop('requester_pays', False) ###### url = event.get('url') params = event.get('search', {}) max_items_batch = event.get('max_items_batch', 15000) sleep = event.get('sleep', None) # search API search = Search(api_url=url, **params) logger.debug(f"Searching {url}") found = search.found() logger.debug(f"Total items found: {found}") if found <= MAX_ITEMS_REQUEST: return run(params, url, sleep=sleep) elif hasattr(context, "invoked_function_arn"): nbatches = int(found / max_items_batch) + 1 if nbatches == 1: submit_batch_job(event, context.invoked_function_arn, definition='lambda-as-batch') else: for request in split_request(params, nbatches): event['search'] = request submit_batch_job(event, context.invoked_function_arn, definition='lambda-as-batch') logger.info(f"Submitted {nbatches} batches") return else: run(params, url, sleep=sleep)
def post(self): body = request.get_json() geom = { "type": "Polygon", "coordinates": [[[-66.3958740234375, 43.305193797650546], [-64.390869140625, 43.305193797650546], [-64.390869140625, 44.22945656830167], [-66.3958740234375, 44.22945656830167], [-66.3958740234375, 43.305193797650546]]] } print(body) cloud = "eo:cloud_cover<" + "5" #body['cloud'] time = "2018-02-01/2018-02-10" #body["time"] search = Search(intersects=geom, time=time, property=[cloud]) items = search.items() print(search.found()) for item in items: print(item.assets["thumbnail"]["href"]) return 200
def stac_api_to_odc( dc: Datacube, products: list, limit: int, update: bool, allow_unsafe: bool, config: dict, **kwargs, ) -> Tuple[int, int]: # QA the BBOX if config["bbox"]: assert ( len(config["bbox"]) == 4 ), "Bounding box must be of the form lon-min,lat-min,lon-max,lat-max" # QA the search srch = Search().search(**config) n_items = srch.found() logging.info("Found {} items to index".format(n_items)) if n_items > 10000: logging.warning( "More than 10,000 items were returned by your query, which is greater than the API limit" ) if n_items == 0: logging.warning("Didn't find any items, finishing.") return 0, 0 # Get a generator of (stac, uri, relative_uri) tuples potential_items = get_items(srch, limit) # Get a generator of (dataset, uri) doc2ds = Doc2Dataset(dc.index, **kwargs) datasets = transform_items(doc2ds, potential_items) # Do the indexing of all the things return index_update_datasets(dc, datasets, update, allow_unsafe)
from satsearch import Search from satstac import Item search = Search(bbox=[-110, 39.5, -105, 40.5]) print('bbox search: %s items' % search.found()) search = Search(time='2018-02-12T00:00:00Z/2018-03-18T12:31:12Z') print('time search: %s items' % search.found()) search = Search(query={'eo:cloud_cover': {'lt': 10}}) print('cloud_cover search: %s items' % search.found()) search = Search(bbox=[-110, 39.5, -105, 40.5], time='2018-02-12T00:00:00Z/2018-03-18T12:31:12Z', query={'eo:cloud_cover': { 'lt': 10 }}) print('%s items' % search.found()) geom = { "type": "Polygon", "coordinates": [[[-66.3958740234375, 43.305193797650546], [-64.390869140625, 43.305193797650546], [-64.390869140625, 44.22945656830167], [-66.3958740234375, 44.22945656830167], [-66.3958740234375, 43.305193797650546]]] } search = Search(intersects=geom) print('intersects search: %s items' % search.found())
import re # os.environ['SATUTILS_API_URL'] = "https://sat-api.developmentseed.org/" # search = Search(bbox=[-121.671912, 39.789597, -121.566920, 39.720973], # datetime='2018-10-30/2018-11-07', # property=["eo:cloud_cover < 30"]) # print('bbox search: %s items' % search.found()) # # items = search.items() # items.save('camp-fire.json') # items.download('B4') # items.download('B3') # items.download('B2') #print(items.summary()) search = Search(bbox=[-110, 39.5, -105, 40.5]) print('bbox search: %s items' % search.found()) items = search.items() print(type(items)) items.save('test.json') items.download('B4') items.download('B3') items.download('B2') files = os.listdir() for file in files: if 'B2.TIF' in file: os.rename(file, 'B2.TIF') if 'B3.TIF' in file: os.rename(file, 'B3.TIF') if 'B4.TIF' in file: os.rename(file, 'B4.TIF')
maxdate1 = f'{(t+dt).year:04}-{(t+dt).month:02}-{(t+dt).day:02}' geojson = mapping(shape(geojson).buffer(1e-5)) query = { "url": "https://earth-search.aws.element84.com/v0", "intersects": copy.copy(geojson), "query": { "eo:cloud_cover": { "lte": args.maxclouds, "gte": args.minclouds, } }, "datetime": f"{mindate1}/{maxdate1}", "sort": [ { "field": "datetime", "direction": ">", }, ], "collections": ["sentinel-s2-l2a"], "limit": args.limit, } search = Search(**query) if search.found() > 0: print(f'*date={t} points={len(ps)}') items = search.items(limit=args.limit) items.save(f'{year:04}_{month:02}_{day:02}.result') else: print(f' date={t} points={len(ps)}')