def run(self):
if not self.isRunning:
self.isRunning = True
items_count = 0
self.search_is_started.emit(
"[" + str(datetime.datetime.now().strftime("%H:%M:%S")) + "]" +
" Ищу...")
# is really shit-shit code,
# но ребятишки, сделавшие API, видимо не подумали о том,
# что может потребоваться передавать параметры через OR (||)
if isinstance(self.query, dict):
if self.intersects is not None:
search = Search(intersects=self.intersects,
time=self.time,
query=self.query)
else:
search = Search(time=self.time, query=self.query)
items_count = search.found()
self.files_are_found.emit(items_count,
self.query["collection"]["eq"])
elif isinstance(self.query, list):
for single_query in self.query:
if self.intersects is not None:
search = Search(intersects=self.intersects,
time=self.time,
query=single_query)
else:
search = Search(time=self.time, query=single_query)
items_count += search.found()
self.files_are_found.emit(items_count,
self.query[0]["collection"]["eq"])
def run(params, url, sleep=None):
search = Search(api_url=url, **params)
logger.debug(f"Searching {url}")
found = search.found()
logger.debug(f"Total items found: {found}")
if found < MAX_ITEMS_REQUEST:
logger.info(f"Making single request for {found} items")
items = search.items()
for i, item in enumerate(items):
resp = SNS_CLIENT.publish(TopicArn=SNS_TOPIC,
Message=json.dumps(item._data))
if (i % 500) == 0:
logger.debug(f"Added {i+1} items to Cirrus")
#if resp['StatusCode'] != 200:
# raise Exception("Unable to publish")
if sleep:
time.sleep(sleep)
logger.debug(f"Published {len(items)} items to {SNS_TOPIC}")
else:
# bisection
nbatches = 2
logger.info(
f"Too many Items for single request, splitting into {nbatches} batches by date range"
)
for params in split_request(params, nbatches):
run(params, url)
def index_data(bbox):
start_date = '2020-05-01'
end_date = '2021-01-06'
print(bbox)
collections = ['sentinel-s2-l2a-cogs']
config = {
'collections': collections,
'bbox': bbox,
'datetime': f"{start_date}/{end_date}"
}
STAC_API_URL = 'https://explorer.sandbox.dea.ga.gov.au/stac/'
os.environ['STAC_API_URL'] = STAC_API_URL
srch = Search().search(**config)
found_items = srch.found()
print(f"Found {found_items} items that can be indexed")
dc = datacube.Datacube()
indexed, failed = stac_api_to_odc(dc, 's2_l2a', None, False, False, config)
print(f"Indexed {indexed} out of {found_items} with {failed} failures.")
def lambda_handler(event, context={}):
logger.debug('Event: %s' % json.dumps(event))
url = event.get('url')
params = event.get('search', {})
max_items_batch = event.get('max_items_batch', 15000)
sleep = event.get('sleep', None)
process = event.get('process', None)
# search API
search = Search(url=url, **params)
logger.debug(f"Searching {url}")
found = search.found()
logger.debug(f"Total items found: {found}")
if found <= MAX_ITEMS_REQUEST:
return run(params, url, sleep=sleep, process=process)
elif hasattr(context, "invoked_function_arn"):
nbatches = int(found / max_items_batch) + 1
if nbatches == 1:
submit_batch_job(event,
context.invoked_function_arn,
definition='lambda-as-batch')
else:
for request in split_request(params, nbatches):
event['search'] = request
submit_batch_job(event,
context.invoked_function_arn,
definition='lambda-as-batch')
logger.info(f"Submitted {nbatches} batches")
return
else:
run(params, url, sleep=sleep, process=process)
def stac_search(extent, start_date, end_date):
""" Convert lat, lon to pathrows """
logger.info("Querying STAC for area: {} and times: {} - {} (UTC)".format(
extent, start_date, end_date))
srch = Search(bbox=extent,
time='{}T00:00:00Z/{}T23:59:59Z'.format(
start_date, end_date),
url="https://sat-api.developmentseed.org/stac/search")
try:
logger.info("Found {} items".format(srch.found()))
return srch
except KeyError:
return None
def lambda_handler(event, context={}):
logger.debug('Event: %s' % json.dumps(event))
# if this is batch, output to stdout
if not hasattr(context, "invoked_function_arn"):
logger.addHandler(logging.StreamHandler())
# parse input
#s3urls = event['s3urls']
#suffix = event.get('suffix', 'json')
#credentials = event.get('credentials', {})
#requester_pays = credentials.pop('requester_pays', False)
######
url = event.get('url')
params = event.get('search', {})
max_items_batch = event.get('max_items_batch', 15000)
sleep = event.get('sleep', None)
# search API
search = Search(api_url=url, **params)
logger.debug(f"Searching {url}")
found = search.found()
logger.debug(f"Total items found: {found}")
if found <= MAX_ITEMS_REQUEST:
return run(params, url, sleep=sleep)
elif hasattr(context, "invoked_function_arn"):
nbatches = int(found / max_items_batch) + 1
if nbatches == 1:
submit_batch_job(event,
context.invoked_function_arn,
definition='lambda-as-batch')
else:
for request in split_request(params, nbatches):
event['search'] = request
submit_batch_job(event,
context.invoked_function_arn,
definition='lambda-as-batch')
logger.info(f"Submitted {nbatches} batches")
return
else:
run(params, url, sleep=sleep)
def post(self):
body = request.get_json()
geom = {
"type":
"Polygon",
"coordinates": [[[-66.3958740234375, 43.305193797650546],
[-64.390869140625, 43.305193797650546],
[-64.390869140625, 44.22945656830167],
[-66.3958740234375, 44.22945656830167],
[-66.3958740234375, 43.305193797650546]]]
}
print(body)
cloud = "eo:cloud_cover<" + "5" #body['cloud']
time = "2018-02-01/2018-02-10" #body["time"]
search = Search(intersects=geom, time=time, property=[cloud])
items = search.items()
print(search.found())
for item in items:
print(item.assets["thumbnail"]["href"])
return 200
def stac_api_to_odc(
dc: Datacube,
products: list,
limit: int,
update: bool,
allow_unsafe: bool,
config: dict,
**kwargs,
) -> Tuple[int, int]:
# QA the BBOX
if config["bbox"]:
assert (
len(config["bbox"]) == 4
), "Bounding box must be of the form lon-min,lat-min,lon-max,lat-max"
# QA the search
srch = Search().search(**config)
n_items = srch.found()
logging.info("Found {} items to index".format(n_items))
if n_items > 10000:
logging.warning(
"More than 10,000 items were returned by your query, which is greater than the API limit"
)
if n_items == 0:
logging.warning("Didn't find any items, finishing.")
return 0, 0
# Get a generator of (stac, uri, relative_uri) tuples
potential_items = get_items(srch, limit)
# Get a generator of (dataset, uri)
doc2ds = Doc2Dataset(dc.index, **kwargs)
datasets = transform_items(doc2ds, potential_items)
# Do the indexing of all the things
return index_update_datasets(dc, datasets, update, allow_unsafe)
from satsearch import Search
from satstac import Item
search = Search(bbox=[-110, 39.5, -105, 40.5])
print('bbox search: %s items' % search.found())
search = Search(time='2018-02-12T00:00:00Z/2018-03-18T12:31:12Z')
print('time search: %s items' % search.found())
search = Search(query={'eo:cloud_cover': {'lt': 10}})
print('cloud_cover search: %s items' % search.found())
search = Search(bbox=[-110, 39.5, -105, 40.5],
time='2018-02-12T00:00:00Z/2018-03-18T12:31:12Z',
query={'eo:cloud_cover': {
'lt': 10
}})
print('%s items' % search.found())
geom = {
"type":
"Polygon",
"coordinates": [[[-66.3958740234375, 43.305193797650546],
[-64.390869140625, 43.305193797650546],
[-64.390869140625, 44.22945656830167],
[-66.3958740234375, 44.22945656830167],
[-66.3958740234375, 43.305193797650546]]]
}
search = Search(intersects=geom)
print('intersects search: %s items' % search.found())
import re
# os.environ['SATUTILS_API_URL'] = "https://sat-api.developmentseed.org/"
# search = Search(bbox=[-121.671912, 39.789597, -121.566920, 39.720973],
# datetime='2018-10-30/2018-11-07',
# property=["eo:cloud_cover < 30"])
# print('bbox search: %s items' % search.found())
#
# items = search.items()
# items.save('camp-fire.json')
# items.download('B4')
# items.download('B3')
# items.download('B2')
#print(items.summary())
search = Search(bbox=[-110, 39.5, -105, 40.5])
print('bbox search: %s items' % search.found())
items = search.items()
print(type(items))
items.save('test.json')
items.download('B4')
items.download('B3')
items.download('B2')
files = os.listdir()
for file in files:
if 'B2.TIF' in file:
os.rename(file, 'B2.TIF')
if 'B3.TIF' in file:
os.rename(file, 'B3.TIF')
if 'B4.TIF' in file:
os.rename(file, 'B4.TIF')
maxdate1 = f'{(t+dt).year:04}-{(t+dt).month:02}-{(t+dt).day:02}'
geojson = mapping(shape(geojson).buffer(1e-5))
query = {
"url": "https://earth-search.aws.element84.com/v0",
"intersects": copy.copy(geojson),
"query": {
"eo:cloud_cover": {
"lte": args.maxclouds,
"gte": args.minclouds,
}
},
"datetime": f"{mindate1}/{maxdate1}",
"sort": [
{
"field": "datetime",
"direction": ">",
},
],
"collections": ["sentinel-s2-l2a"],
"limit": args.limit,
}
search = Search(**query)
if search.found() > 0:
print(f'*date={t} points={len(ps)}')
items = search.items(limit=args.limit)
items.save(f'{year:04}_{month:02}_{day:02}.result')
else:
print(f' date={t} points={len(ps)}')