def handler(payload, context={}, local=None):
""" Handle Cirrus payload (STAC Process Catalog) """
# get catalog
catalog = Catalog.from_payload(payload)
# configure logger
logger = get_task_logger(f"{__name__}.{TASK_NAME}", catalog=catalog)
# these are any optional parameter provided for this task
config = catalog['process']['tasks'].get(TASK_NAME, {})
# these are general options used when uploading output data to s3
outopts = payload['process'].get('output_options', {})
# validation - add specific checks on input
# e.g., if task operates on one and only Item use this:
assert (len(catalog['features']) == 1)
item = catalog['features'][0]
# create temporary work directory if not running locally
tmpdir = mkdtemp() if local is None else local
outpath = op.join(tmpdir, 'output')
os.makedirs(outpath, exist_ok=True)
try:
# main logic - replace with own
# download asset, e.g. a thumbnail
item = download_item_assets(item, path=outpath, assets=['thumbnail'])
# do something, e.g. modify asset, create new asset
# item['assets']['asset2'] = create_new_asset(item)
# upload new assets
if local is not None:
item = upload_item_assets(item, assets=['asset2'], **outopts)
# recommended to add derived_from link
links = [l['href'] for l in item['links'] if l['rel'] == 'self']
if len(links) == 1:
# add derived from link
item['links'].append({
'title': 'Source STAC Item',
'rel': 'derived_from',
'href': links[0],
'type': 'application/json'
})
catalog['features'][0] = item
except Exception as err:
msg = f"**task** failed: {err}"
logger.error(msg, exc_info=True)
raise Exception(msg)
finally:
# remove work directory if not running locally
if local is None:
logger.debug('Removing work directory %s' % tmpdir)
rmtree(tmpdir)
return catalog
def test_upload_item_assets(self):
item = self.get_test_item()
path_template = 's3://testbucket/${id}/test'
assets = ['local']
new_item = transfer.upload_item_assets(item,
assets=assets,
path_template=path_template,
s3_urls=True,
region_name='us-west-2')
for k in assets:
assert (new_item['assets'][k]['href'].startswith('s3://'))
assert (s3().exists(new_item['assets'][k]['href']))
def lambda_handler(payload, context={}):
# if this is batch, output to stdout
if not hasattr(context, "invoked_function_arn"):
logger.addHandler(logging.StreamHandler())
logger.debug('Payload: %s' % json.dumps(payload))
catalog = Catalogs.from_payload(payload)[0]
# assign proper collection names based on ID
catalog.assign_collections()
# TODO - make this more general for more items/collections
item = catalog['features'][0] #, collection=catalog['collections'][0])
# configuration options
config = catalog['process']['tasks'].get('copy-assets', {})
outopts = catalog['process'].get('output_options', {})
# asset config
assets = config.get('assets', item['assets'].keys())
drop_assets = config.get('drop_assets', [])
# drop specified assets
for asset in [a for a in drop_assets if a in item['assets'].keys()]:
logger.debug(f'Dropping asset {asset}')
item['assets'].pop(asset)
if type(assets) is str and assets == 'ALL':
assets = item['assets'].keys()
# create temporary work directory
tmpdir = mkdtemp()
try:
# copy specified assets
_assets = [a for a in assets if a in item['assets'].keys()]
for asset in _assets:
item = download_item_assets(item, path=tmpdir, assets=[asset])
item = upload_item_assets(item, assets=[asset], **outopts)
# replace item in catalog
catalog['features'][0] = item
except Exception as err:
msg = f"copy-assets: failed processing {catalog['id']} ({err})"
logger.error(msg)
logger.error(format_exc())
raise Exception(msg) from err
finally:
# remove work directory....very important for Lambdas!
logger.debug('Removing work directory %s' % tmpdir)
rmtree(tmpdir)
return catalog
def handler(payload, context={}):
catalog = Catalog.from_payload(payload)
logger = get_task_logger(f"{__name__}.copy-assets", catalog=catalog)
# TODO - make this more general for more items/collections
item = catalog['features'][0] #, collection=catalog['collections'][0])
# configuration options
config = catalog['process']['tasks'].get('copy-assets', {})
outopts = catalog['process'].get('output_options', {})
# asset config
assets = config.get('assets', item['assets'].keys())
drop_assets = config.get('drop_assets', [])
# drop specified assets
for asset in [a for a in drop_assets if a in item['assets'].keys()]:
logger.debug(f'Dropping asset {asset}')
item['assets'].pop(asset)
if type(assets) is str and assets == 'ALL':
assets = item['assets'].keys()
# create temporary work directory
tmpdir = mkdtemp()
try:
# copy specified assets
_assets = [a for a in assets if a in item['assets'].keys()]
for asset in _assets:
item = download_item_assets(item, path=tmpdir, assets=[asset])
item = upload_item_assets(item, assets=[asset], **outopts)
# replace item in catalog
catalog['features'][0] = item
except Exception as err:
msg = f"copy-assets: failed processing {catalog['id']} ({err})"
logger.error(msg, exc_info=True)
raise Exception(msg) from err
finally:
# remove work directory....very important for Lambdas!
logger.debug('Removing work directory %s' % tmpdir)
rmtree(tmpdir)
return catalog
def lambda_handler(payload, context={}):
# if this is batch, output to stdout
if not hasattr(context, "invoked_function_arn"):
logger.addHandler(logging.StreamHandler())
logger.debug('Payload: %s' % json.dumps(payload))
catalog = Catalogs.from_payload(payload)[0]
# get step configuration
config = catalog['process']['tasks'].get('add-preview', {})
outopts = catalog['process'].get('output_options', {})
assets = config.pop('assets', None)
thumb = config.pop('thumbnail', False)
if assets is None:
msg = f"add-preview: no asset specified for preview, skipping {catalog['id']}"
logger.error(msg)
raise Exception(msg)
# create temporary work directory
tmpdir = tempfile.mkdtemp()
items = []
for item in catalog['features']:
# find asset to use for preview
asset = None
for a in assets:
if a in item['assets']:
asset = a
break
if asset is None:
msg = f"add-preview: no asset specified for preview, skipping {item['id']}"
logger.warning(msg)
return item
try:
# keep original href
href = item['assets'][asset]['href']
# download asset
item = download_item_assets(item, path=tmpdir, assets=[asset])
# add preview to item
add_preview(item, item['assets'][asset]['href'], **config)
if thumb:
# add thumbnail to item
add_thumbnail(item, item['assets']['preview']['href'])
# put back original href
item['assets'][asset]['href'] = href
# set item in return catalog to this new item
#catalog['features'][0] = item._data
# upload these new assets
item = upload_item_assets(item,
assets=['preview', 'thumbnail'],
**outopts)
items.append(item)
except Exception as err:
msg = f"add-preview: failed creating preview/thumbnail for {catalog['id']} ({err})"
logger.error(msg)
logger.error(format_exc())
# remove work directory....very important for Lambdas!
logger.debug('Removing work directory %s' % tmpdir)
shutil.rmtree(tmpdir)
raise Exception(msg) from err
catalog['features'] = items
# remove work directory....very important for Lambdas!
logger.debug('Removing work directory %s' % tmpdir)
shutil.rmtree(tmpdir)
return catalog
def handler(payload, context={}):
catalog = Catalog.from_payload(payload)
logger = get_task_logger(f"{__name__}.add-preview", catalog=catalog)
# get step configuration
config = catalog['process']['tasks'].get('add-preview', {})
outopts = catalog['process'].get('output_options', {})
assets = config.pop('assets', None)
thumb = config.pop('thumbnail', False)
config.pop('batch')
if assets is None:
msg = f"add-preview: no asset specified for preview"
logger.error(msg)
raise Exception(msg)
# create temporary work directory
tmpdir = tempfile.mkdtemp()
items = []
for item in catalog['features']:
# find asset to use for preview
asset = None
for a in assets:
if a in item['assets']:
asset = a
break
if asset is None:
msg = f"add-preview: no available asset for preview"
logger.warning(msg)
items.append(item)
continue
try:
# keep original href
href = item['assets'][asset]['href']
# download asset
item = download_item_assets(item, path=tmpdir, assets=[asset])
filename = item['assets'][asset]['href']
# add preview to item
item['assets']['preview'] = create_preview(filename, logger,
**config)
if thumb:
# add thumbnail to item
item['assets']['thumbnail'] = create_thumbnail(
item['assets']['preview']['href'], logger)
# put back original href
item['assets'][asset]['href'] = href
# upload these new assets
item = upload_item_assets(item,
assets=['preview', 'thumbnail'],
**outopts)
items.append(item)
except Exception as err:
msg = f"add-preview: failed creating preview/thumbnail ({err})"
logger.error(msg, exc_info=True)
# remove work directory....very important for Lambdas!
shutil.rmtree(tmpdir)
raise Exception(msg) from err
# remove work directory....very important for Lambdas!
shutil.rmtree(tmpdir)
# return new items
catalog['features'] = items
return catalog
def handler(payload, context={}):
catalog = Catalog.from_payload(payload)
logger = get_task_logger(f"{__name__}.convert-to-cog", catalog=catalog)
# TODO - make this more general for more items/collections
item = catalog['features'][0] #, collection=catalog['collections'][0])
# configuration options
config = catalog['process']['tasks'].get('convert-to-cog', {})
outopts = catalog['process'].get('output_options', {})
assets = config.get('assets')
# create temporary work directory
tmpdir = mkdtemp()
try:
asset_keys = [a for a in assets if a in item['assets'].keys()]
for asset in asset_keys:
logger.info(f"Converting {asset} to COG")
# download asset
item = download_item_assets(item, path=tmpdir, assets=[asset])
# cogify
fn = item['assets'][asset]['href']
fnout = cogify(fn, op.splitext(fn)[0] + '.tif', **assets[asset])
item['assets'][asset]['href'] = fnout
item['assets'][asset][
'type'] = "image/tiff; application=geotiff; profile=cloud-optimized"
with rasterio.open(fnout) as src:
item['assets'][asset]['proj:shape'] = src.shape
item['assets'][asset]['proj:transform'] = src.transform
# upload assets
item = upload_item_assets(item, assets=[asset], **outopts)
# cleanup files
if op.exists(fn):
remove(fn)
if op.exists(fnout):
remove(fnout)
# add derived_from link
links = [l['href'] for l in item['links'] if l['rel'] == 'self']
if len(links) == 1:
# add derived from link
item['links'].append({
'title': 'Source STAC Item',
'rel': 'derived_from',
'href': links[0],
'type': 'application/json'
})
# drop any specified assets
for asset in [
a for a in config.get('drop_assets', [])
if a in item['assets'].keys()
]:
item['assets'].pop(asset)
catalog['features'][0] = item
except CRSError as err:
msg = f"convert-to-cog: invalid CRS ({err})"
logger.error(msg, exc_info=True)
raise InvalidInput(msg)
except s3_sessions[list(s3_sessions)[0]].s3.exceptions.NoSuchKey as err:
msg = f"convert-to-cog: failed fetching {asset} asset ({err})"
logger.error(msg, exc_info=True)
raise InvalidInput(msg)
except Exception as err:
msg = f"convert-to-cog: failed creating COGs ({err})"
logger.error(msg, exc_info=True)
raise Exception(msg)
finally:
# remove work directory....very important for Lambdas!
logger.debug('Removing work directory %s' % tmpdir)
rmtree(tmpdir)
return catalog
def lambda_handler(payload, context={}):
# if this is batch, output to stdout
if not hasattr(context, "invoked_function_arn"):
logger.addHandler(StreamHandler())
logger.debug('Payload: %s' % json.dumps(payload))
catalog = Catalogs.from_payload(payload)[0]
# TODO - make this more general for more items/collections
item = catalog['features'][0] #, collection=catalog['collections'][0])
# configuration options
config = catalog['process']['tasks'].get('convert-to-cog', {})
outopts = catalog['process'].get('output_options', {})
assets = config.get('assets')
# create temporary work directory
tmpdir = mkdtemp()
try:
asset_keys = [a for a in assets if a in item['assets'].keys()]
for asset in asset_keys:
# download asset
item = download_item_assets(item, path=tmpdir, assets=[asset])
logger.debug(f"Downloaded item: {json.dumps(item)}")
# cogify
fn = item['assets'][asset]['href']
fnout = cogify(fn, op.splitext(fn)[0] + '.tif', **assets[asset])
item['assets'][asset]['href'] = fnout
item['assets'][asset][
'type'] = "image/tiff; application=geotiff; profile=cloud-optimized"
with rasterio.open(fnout) as src:
item['assets'][asset]['proj:shape'] = src.shape
item['assets'][asset]['proj:transform'] = src.transform
# upload assets
item = upload_item_assets(item, assets=[asset], **outopts)
# cleanup files
if op.exists(fn):
remove(fn)
if op.exists(fnout):
remove(fnout)
# add derived_from link
links = [l['href'] for l in item['links'] if l['rel'] == 'self']
if len(links) == 1:
# add derived from link
item['links'].append({
'title': 'Source STAC Item',
'rel': 'derived_from',
'href': links[0],
'type': 'application/json'
})
# drop any specified assets
for asset in [
a for a in config.get('drop_assets', [])
if a in item['assets'].keys()
]:
logger.info(f"Dropping {asset}")
item['assets'].pop(asset)
catalog['features'][0] = item
except CRSError as err:
msg = f"convert-to-cog: invalid CRS for {catalog['id']} ({err})"
logger.error(msg)
logger.error(format_exc())
raise InvalidInput(msg)
except Exception as err:
msg = f"convert-to-cog: failed creating COGs for {catalog['id']} ({err})"
logger.error(msg)
logger.error(format_exc())
raise Exception(msg) from err
finally:
# remove work directory....very important for Lambdas!
logger.debug('Removing work directory %s' % tmpdir)
rmtree(tmpdir)
return catalog