def generate_scenes(self):
"""Create a Scene and associated Image for each GeoTiff in self.s3_path
Returns:
Generator of Scenes
"""
s3 = boto3.resource('s3')
for infile in self.files:
# We can't use the temp file as a context manager because it'll be opened/closed multiple
# times and by default is deleted when it's closed. So we use try/finally to ensure that
# it gets cleaned up.
bucket_name, key = s3_bucket_and_key_from_url(infile)
filename = os.path.basename(key)
logger.info('Downloading %s => %s', infile, filename)
bucket = s3.Bucket(bucket_name)
with get_tempdir() as tempdir:
tmp_fname = os.path.join(tempdir, filename)
bucket.download_file(key, tmp_fname)
cog.add_overviews(tmp_fname)
cog_path = cog.convert_to_cog(tmp_fname, tempdir)
scene = self.create_geotiff_scene(tmp_fname, os.path.splitext(filename)[0])
scene.ingestLocation = upload_tifs([cog_path], self.owner, scene.id)[0]
images = [self.create_geotiff_image(
tmp_fname, urllib.unquote(scene.ingestLocation), scene, cog_path
)]
scene.thumbnails = []
scene.images = images
yield scene
def create_cog(image_locations, scene, same_path=False):
"""
Args:
image_locations (List[(uri, filename)]): Used to fetch source imagery for the scene for processing
scene (Scene): Scene to create COG from
same_path (boolean): Output to the same path that it was downloaded from
Returns:
Scene: The mutated scene. Must call update() on it to be reflected on the API
Raises:
Exception: Any exceptions here are unrecoverable.
"""
with get_tempdir() as local_dir:
dsts = [os.path.join(local_dir, fname) for _, fname in image_locations]
cog.fetch_imagery(image_locations, local_dir)
warped_paths = cog.warp_tifs(dsts, local_dir)
merged_tif = cog.merge_tifs(warped_paths, local_dir)
cog.add_overviews(merged_tif)
cog_path = cog.convert_to_cog(merged_tif, local_dir)
if same_path:
updated_scene = upload_tif(
cog_path, scene,
os.path.join('user-uploads', scene.owner, '{}_COG.tif'.format(scene.id)),
os.path.join('user-uploads', urllib.quote_plus(scene.owner), '{}_COG.tif'.format(scene.id))
)
else:
updated_scene = upload_tif(cog_path, scene)
os.remove(cog_path)
return updated_scene
def generate_scenes(self):
"""Create a Scene and associated Image for each GeoTiff in self.s3_path
Returns:
Generator of Scenes
"""
s3 = boto3.resource('s3')
for infile in self.files:
# We can't use the temp file as a context manager because it'll be opened/closed multiple
# times and by default is deleted when it's closed. So we use try/finally to ensure that
# it gets cleaned up.
bucket_name, key = s3_bucket_and_key_from_url(infile)
filename = os.path.basename(key)
logger.info('Downloading %s => %s', infile, filename)
bucket = s3.Bucket(bucket_name)
with get_tempdir() as tempdir:
tmp_fname = os.path.join(tempdir, filename)
bucket.download_file(key, tmp_fname)
cog.add_overviews(tmp_fname)
cog_path = cog.convert_to_cog(tmp_fname, tempdir)
scene = self.create_geotiff_scene(
tmp_fname,
os.path.splitext(filename)[0])
scene.ingestLocation = upload_tifs([cog_path], self.owner,
scene.id)[0]
images = [
self.create_geotiff_image(
tmp_fname, urllib.unquote(scene.ingestLocation), scene,
cog_path)
]
scene.thumbnails = []
scene.images = images
yield scene
def create_cog(image_locations, scene):
with get_tempdir() as local_dir:
dsts = [os.path.join(local_dir, fname) for _, fname in image_locations]
cog.fetch_imagery(image_locations, local_dir)
warped_paths = cog.warp_tifs(dsts, local_dir)
merged_tif = cog.merge_tifs(warped_paths, local_dir)
cog.add_overviews(merged_tif)
cog_path = cog.convert_to_cog(merged_tif, local_dir)
updated_scene = upload_tif(cog_path, scene)
updated_scene.update()
def copy_asset_to_s3(self, prefix, planet_id):
"""Make the Planet tif available to Rater Foundry
This function downloads the basic analytic tif from planet,
then reuploads it to s3 in a directory for planet files.
Args:
planet_id (str): id of the planet image to download
Returns:
dict: geojson for the overview of the planet tif
"""
split_id = planet_id.split(':')
item_type = split_id[0]
item_id = split_id[1]
if len(split_id) == 2:
asset_type = None
elif len(split_id) == 3:
asset_type = split_id[2]
logger.info('Retrieving item type %s with id %s', item_type, item_id)
item = self.get_item(item_id, item_type)
item_id = item['id']
assets = self.get_assets_by_id(item_id, item_type)
asset_type = PlanetSceneFactory.get_asset_type(assets, asset_type)
updated_assets = self.activate_asset_and_wait(asset_type, assets,
item_id, item_type)
temp_tif_file = self.download_planet_tif(prefix, asset_type,
updated_assets, item_id)
full_path_to_temp_tif = os.path.join(prefix, temp_tif_file)
cog.add_overviews(full_path_to_temp_tif)
cog_path = cog.convert_to_cog(full_path_to_temp_tif, prefix)
bucket, s3_path = self.upload_planet_tif(asset_type, item_id,
item_type, cog_path)
analytic_xml = self.get_analytic_xml(assets, item_id, item_type)
reflectance_coefficients = PlanetSceneFactory.get_reflectance_coefficients(
analytic_xml)
item['properties'].update(reflectance_coefficients)
item['added_props'] = {}
item['added_props']['localPath'] = cog_path
item['added_props']['s3Location'] = 's3://{}/{}'.format(
bucket, s3_path)
item['added_props']['asset_type'] = asset_type
# Return the json representation of the item
return item, temp_tif_file
def process_to_cog(prefix, gcs_prefix, landsat_id, config):
logger.info('Fetching all bands')
for band in config.bands.keys():
fetch_band(prefix, gcs_prefix, band, landsat_id)
cog_fname = '{}_COG.tif'.format(landsat_id)
stacked_fname = '{}_STACKED.tif'.format(landsat_id)
filenames = {
'COG': os.path.join(prefix, cog_fname),
'STACKED': os.path.join(prefix, stacked_fname)
}
local_paths = sorted(glob.glob('/{}/{}*.TIF'.format(prefix, landsat_id)))
warped_paths = cog.warp_tifs(local_paths, prefix)
merged = cog.merge_tifs(warped_paths, prefix)
cog.add_overviews(merged)
cog_path = cog.convert_to_cog(merged, prefix)
shutil.move(cog_path, filenames['COG'])
return (filenames['COG'], cog_fname)
def process_to_cog(prefix, gcs_prefix, landsat_id, config):
logger.info('Fetching all bands')
for band in config.bands.keys():
fetch_band(prefix, gcs_prefix, band, landsat_id)
cog_fname = '{}_COG.tif'.format(landsat_id)
stacked_fname = '{}_STACKED.tif'.format(landsat_id)
filenames = {
'COG': os.path.join(prefix, cog_fname),
'STACKED': os.path.join(prefix, stacked_fname)
}
local_paths = sorted(glob.glob('{}/{}*.TIF'.format(prefix, landsat_id)))
warped_paths = cog.warp_tifs(local_paths, prefix)
merged = cog.merge_tifs(warped_paths, prefix)
cog.add_overviews(merged)
cog_path = cog.convert_to_cog(merged, prefix)
shutil.move(cog_path, filenames['COG'])
return (filenames['COG'], cog_fname)
def copy_asset_to_s3(self, prefix, planet_id):
"""Make the Planet tif available to Rater Foundry
This function downloads the basic analytic tif from planet,
then reuploads it to s3 in a directory for planet files.
Args:
planet_id (str): id of the planet image to download
Returns:
dict: geojson for the overview of the planet tif
"""
split_id = planet_id.split(':')
item_type = split_id[0]
item_id = split_id[1]
if len(split_id) == 2:
asset_type = None
elif len(split_id) == 3:
asset_type = split_id[2]
logger.info('Retrieving item type %s with id %s', item_type, item_id)
item = self.get_item(item_id, item_type)
item_id = item['id']
assets = self.get_assets_by_id(item_id, item_type)
asset_type = PlanetSceneFactory.get_asset_type(assets, asset_type)
updated_assets = self.activate_asset_and_wait(asset_type, assets, item_id, item_type)
temp_tif_file = self.download_planet_tif(prefix, asset_type, updated_assets, item_id)
full_path_to_temp_tif = os.path.join(prefix, temp_tif_file)
cog.add_overviews(full_path_to_temp_tif)
cog_path = cog.convert_to_cog(full_path_to_temp_tif, prefix)
bucket, s3_path = self.upload_planet_tif(asset_type, item_id, item_type, cog_path)
analytic_xml = self.get_analytic_xml(assets, item_id, item_type)
reflectance_coefficients = PlanetSceneFactory.get_reflectance_coefficients(analytic_xml)
item['properties'].update(reflectance_coefficients)
item['added_props'] = {}
item['added_props']['localPath'] = cog_path
item['added_props']['s3Location'] = 's3://{}/{}'.format(bucket, s3_path)
item['added_props']['asset_type'] = asset_type
# Return the json representation of the item
return item, temp_tif_file
def create_cog(image_locations, scene, same_path=False):
with get_tempdir() as local_dir:
dsts = [os.path.join(local_dir, fname) for _, fname in image_locations]
cog.fetch_imagery(image_locations, local_dir)
warped_paths = cog.warp_tifs(dsts, local_dir)
merged_tif = cog.merge_tifs(warped_paths, local_dir)
cog.add_overviews(merged_tif)
cog_path = cog.convert_to_cog(merged_tif, local_dir)
if same_path:
updated_scene = upload_tif(
cog_path, scene,
os.path.join('user-uploads', scene.owner, '{}_COG.tif'.format(scene.id)),
os.path.join('user-uploads', urllib.quote_plus(scene.owner), '{}_COG.tif'.format(scene.id))
)
else:
updated_scene = upload_tif(cog_path, scene)
updated_scene.update()
os.remove(cog_path)
def create_cog(image_locations, scene, same_path=False):
with get_tempdir() as local_dir:
dsts = [os.path.join(local_dir, fname) for _, fname in image_locations]
cog.fetch_imagery(image_locations, local_dir)
warped_paths = cog.warp_tifs(dsts, local_dir)
merged_tif = cog.merge_tifs(warped_paths, local_dir)
cog.add_overviews(merged_tif)
cog_path = cog.convert_to_cog(merged_tif, local_dir)
if same_path:
updated_scene = upload_tif(
cog_path, scene,
os.path.join('user-uploads', scene.owner,
'{}_COG.tif'.format(scene.id)),
os.path.join('user-uploads', urllib.quote_plus(scene.owner),
'{}_COG.tif'.format(scene.id)))
else:
updated_scene = upload_tif(cog_path, scene)
updated_scene.update()
os.remove(cog_path)
def create_geotiffs(modis_path, local_dir):
"""Create geotiffs from MODIS HDF
1. Create separate tifs for each band
2. Combine tifs into a single tif
3. Warp tif to web mercator (for quicker access)
4. Generate COG
Args:
modis_path (str): path to modis HDF file
local_dir (str): directory to output tiffs to
"""
logger.info('Preparing to create geotiffs')
post_web_mercator_path = os.path.join(local_dir, 'warp2.tif')
tifs = sorted(hdf_to_geotiffs(modis_path, local_dir))
logger.info('Tifs: %s', '\n'.join(tifs))
warped_paths = cog.warp_tifs(tifs, local_dir)
merged_tif = cog.merge_tifs(warped_paths, local_dir)
warp_tif(merged_tif, post_web_mercator_path)
cog.add_overviews(post_web_mercator_path)
cog_path = cog.convert_to_cog(post_web_mercator_path, local_dir)
return [cog_path]
def create_geotiffs(modis_path, local_dir):
"""Create geotiffs from MODIS HDF
1. Create separate tifs for each band
2. Combine tifs into a single tif
3. Warp tif to web mercator (for quicker access)
4. Generate COG
Args:
modis_path (str): path to modis HDF file
local_dir (str): directory to output tiffs to
"""
logger.info('Preparing to create geotiffs')
post_web_mercator_path = os.path.join(local_dir, 'warp2.tif')
tifs = sorted(hdf_to_geotiffs(modis_path, local_dir))
logger.info('Tifs: %s', '\n'.join(tifs))
warped_paths = cog.warp_tifs(tifs, local_dir)
merged_tif = cog.merge_tifs(warped_paths, local_dir)
warp_tif(merged_tif, post_web_mercator_path)
cog.add_overviews(post_web_mercator_path)
cog_path = cog.convert_to_cog(post_web_mercator_path, local_dir)
return [cog_path]
