def create_cog(image_locations, scene, same_path=False): """ Args: image_locations (List[(uri, filename)]): Used to fetch source imagery for the scene for processing scene (Scene): Scene to create COG from same_path (boolean): Output to the same path that it was downloaded from Returns: Scene: The mutated scene. Must call update() on it to be reflected on the API Raises: Exception: Any exceptions here are unrecoverable. """ with get_tempdir() as local_dir: dsts = [os.path.join(local_dir, fname) for _, fname in image_locations] cog.fetch_imagery(image_locations, local_dir) warped_paths = cog.warp_tifs(dsts, local_dir) merged_tif = cog.merge_tifs(warped_paths, local_dir) cog.add_overviews(merged_tif) cog_path = cog.convert_to_cog(merged_tif, local_dir) if same_path: updated_scene = upload_tif( cog_path, scene, os.path.join('user-uploads', scene.owner, '{}_COG.tif'.format(scene.id)), os.path.join('user-uploads', urllib.quote_plus(scene.owner), '{}_COG.tif'.format(scene.id)) ) else: updated_scene = upload_tif(cog_path, scene) os.remove(cog_path) return updated_scene
def generate_scenes(self): """Create a Scene and associated Image for each GeoTiff in self.s3_path Returns: Generator of Scenes """ s3 = boto3.resource('s3') for infile in self.files: # We can't use the temp file as a context manager because it'll be opened/closed multiple # times and by default is deleted when it's closed. So we use try/finally to ensure that # it gets cleaned up. bucket_name, key = s3_bucket_and_key_from_url(infile) filename = os.path.basename(key) logger.info('Downloading %s => %s', infile, filename) bucket = s3.Bucket(bucket_name) with get_tempdir() as tempdir: tmp_fname = os.path.join(tempdir, filename) bucket.download_file(key, tmp_fname) cog.add_overviews(tmp_fname) cog_path = cog.convert_to_cog(tmp_fname, tempdir) scene = self.create_geotiff_scene(tmp_fname, os.path.splitext(filename)[0]) scene.ingestLocation = upload_tifs([cog_path], self.owner, scene.id)[0] images = [self.create_geotiff_image( tmp_fname, urllib.unquote(scene.ingestLocation), scene, cog_path )] scene.thumbnails = [] scene.images = images yield scene
def generate_scenes(self): """Create a Scene and associated Image for each GeoTiff in self.s3_path Returns: Generator of Scenes """ s3 = boto3.resource('s3') for infile in self.files: # We can't use the temp file as a context manager because it'll be opened/closed multiple # times and by default is deleted when it's closed. So we use try/finally to ensure that # it gets cleaned up. bucket_name, key = s3_bucket_and_key_from_url(infile) filename = os.path.basename(key) logger.info('Downloading %s => %s', infile, filename) bucket = s3.Bucket(bucket_name) with get_tempdir() as tempdir: tmp_fname = os.path.join(tempdir, filename) bucket.download_file(key, tmp_fname) cog.add_overviews(tmp_fname) cog_path = cog.convert_to_cog(tmp_fname, tempdir) scene = self.create_geotiff_scene( tmp_fname, os.path.splitext(filename)[0]) scene.ingestLocation = upload_tifs([cog_path], self.owner, scene.id)[0] images = [ self.create_geotiff_image( tmp_fname, urllib.unquote(scene.ingestLocation), scene, cog_path) ] scene.thumbnails = [] scene.images = images yield scene
def generate_scenes(self): scenes = [] for hdf_url in self.hdf_urls: with get_tempdir() as temp_dir: scene = create_scene(hdf_url, temp_dir, self.owner, self.datasource) scenes.append(scene) return scenes
def generate_scenes(self): scenes = [] for hdf_url in self.hdf_urls: with get_tempdir() as temp_dir: scene = create_scene(hdf_url, temp_dir, self.owner, self.datasource) scenes.append(scene) return scenes
def create_cog(image_locations, scene): with get_tempdir() as local_dir: dsts = [os.path.join(local_dir, fname) for _, fname in image_locations] cog.fetch_imagery(image_locations, local_dir) warped_paths = cog.warp_tifs(dsts, local_dir) merged_tif = cog.merge_tifs(warped_paths, local_dir) cog.add_overviews(merged_tif) cog_path = cog.convert_to_cog(merged_tif, local_dir) updated_scene = upload_tif(cog_path, scene) updated_scene.update()
def generate_scenes(self): scenes = [] for landsat_id in self.upload.files: path_meta = io.base_metadata_for_landsat_id(landsat_id) sensor = path_meta['sensor_id'] config = { 'M': MultiSpectralScannerConfig, 'T': ThematicMapperConfig, 'E': EnhancedThematicMapperConfig }[sensor] with io.get_tempdir() as temp_dir: scene = create_scene(self.upload.owner, temp_dir, landsat_id, config, self.upload.datasource) scenes.append(scene) return scenes
def generate_scenes(self): scenes = [] for landsat_id in self.upload.files: path_meta = io.base_metadata_for_landsat_id(landsat_id) sensor = path_meta['sensor_id'] config = { 'M': MultiSpectralScannerConfig, 'T': ThematicMapperConfig, 'E': EnhancedThematicMapperConfig }[sensor] with io.get_tempdir() as temp_dir: scene = create_scene(self.upload.owner, temp_dir, landsat_id, config, self.upload.datasource) scenes.append(scene) return scenes
def generate_scenes(self): # If this upload is not associated with a project, set the scene's # ingest status to TOBEINGESTED so that scene creation will kick off # an ingest. Otherwise, set the status to NOTINGESTED, so that the status # will be updated when the scene is added to this upload's project for planet_id in set(self.planet_ids): logger.info('Preparing to copy planet asset to s3: %s', planet_id) with get_tempdir() as prefix: planet_feature, temp_tif_file = self.copy_asset_to_s3(prefix, planet_id) planet_key = self.client.auth.value planet_scene = create_planet_scene( planet_feature, self.datasource, planet_key, self.visibility, self.tags, self.owner ) yield planet_scene
def generate_scenes(self): # If this upload is not associated with a project, set the scene's # ingest status to TOBEINGESTED so that scene creation will kick off # an ingest. Otherwise, set the status to NOTINGESTED, so that the status # will be updated when the scene is added to this upload's project for planet_id in set(self.planet_ids): logger.info('Preparing to copy planet asset to s3: %s', planet_id) with get_tempdir() as prefix: planet_feature, temp_tif_file = self.copy_asset_to_s3( prefix, planet_id) planet_key = self.client.auth.value planet_scene = create_planet_scene(planet_feature, self.datasource, planet_key, self.visibility, self.tags, self.owner) yield planet_scene
def reprocess_landsat_h(scene_id): logger.info('Fetching scene to reprocess with correct band order: %s', scene_id) scene = Scene.from_id(scene_id) upload_dst = scene.ingestLocation.split('/')[-1] landsat_id = scene.name gcs_prefix = io.gcs_path_for_landsat_id(landsat_id) path_meta = io.base_metadata_for_landsat_id(landsat_id) sensor = path_meta['sensor_id'] config = { 'M': MultiSpectralScannerConfig, 'T': ThematicMapperConfig, 'E': EnhancedThematicMapperConfig }[sensor] with io.get_tempdir() as prefix: (local_path, _) = process_to_cog(prefix, gcs_prefix, landsat_id, config) upload_file(scene.owner, local_path, upload_dst)
def create_cog(image_locations, scene, same_path=False): with get_tempdir() as local_dir: dsts = [os.path.join(local_dir, fname) for _, fname in image_locations] cog.fetch_imagery(image_locations, local_dir) warped_paths = cog.warp_tifs(dsts, local_dir) merged_tif = cog.merge_tifs(warped_paths, local_dir) cog.add_overviews(merged_tif) cog_path = cog.convert_to_cog(merged_tif, local_dir) if same_path: updated_scene = upload_tif( cog_path, scene, os.path.join('user-uploads', scene.owner, '{}_COG.tif'.format(scene.id)), os.path.join('user-uploads', urllib.quote_plus(scene.owner), '{}_COG.tif'.format(scene.id)) ) else: updated_scene = upload_tif(cog_path, scene) updated_scene.update() os.remove(cog_path)
def create_cog(image_locations, scene, same_path=False): with get_tempdir() as local_dir: dsts = [os.path.join(local_dir, fname) for _, fname in image_locations] cog.fetch_imagery(image_locations, local_dir) warped_paths = cog.warp_tifs(dsts, local_dir) merged_tif = cog.merge_tifs(warped_paths, local_dir) cog.add_overviews(merged_tif) cog_path = cog.convert_to_cog(merged_tif, local_dir) if same_path: updated_scene = upload_tif( cog_path, scene, os.path.join('user-uploads', scene.owner, '{}_COG.tif'.format(scene.id)), os.path.join('user-uploads', urllib.quote_plus(scene.owner), '{}_COG.tif'.format(scene.id))) else: updated_scene = upload_tif(cog_path, scene) updated_scene.update() os.remove(cog_path)