Exemplo n.º 1
0
    def generate_scenes(self):
        """Create a Scene and associated Image for each GeoTiff in self.s3_path
        Returns:
            Generator of Scenes
        """
        s3 = boto3.resource('s3')
        for infile in self.files:
            # We can't use the temp file as a context manager because it'll be opened/closed multiple
            # times and by default is deleted when it's closed. So we use try/finally to ensure that
            # it gets cleaned up.
            bucket_name, key = s3_bucket_and_key_from_url(infile)
            filename = os.path.basename(key)
            logger.info('Downloading %s => %s', infile, filename)
            bucket = s3.Bucket(bucket_name)
            with get_tempdir() as tempdir:
                tmp_fname = os.path.join(tempdir, filename)
                bucket.download_file(key, tmp_fname)
                cog.add_overviews(tmp_fname)
                cog_path = cog.convert_to_cog(tmp_fname, tempdir)
                scene = self.create_geotiff_scene(tmp_fname, os.path.splitext(filename)[0])
                scene.ingestLocation = upload_tifs([cog_path], self.owner, scene.id)[0]
                images = [self.create_geotiff_image(
                    tmp_fname, urllib.unquote(scene.ingestLocation), scene, cog_path
                )]

            scene.thumbnails = []
            scene.images = images
            yield scene
Exemplo n.º 2
0
def create_cog(image_locations, scene, same_path=False):
    """
    Args:
      image_locations (List[(uri, filename)]): Used to fetch source imagery for the scene for processing
      scene (Scene): Scene to create COG from
      same_path (boolean): Output to the same path that it was downloaded from

    Returns:
      Scene: The mutated scene. Must call update() on it to be reflected on the API
    Raises:
      Exception: Any exceptions here are unrecoverable.
    """
    with get_tempdir() as local_dir:
        dsts = [os.path.join(local_dir, fname) for _, fname in image_locations]
        cog.fetch_imagery(image_locations, local_dir)
        warped_paths = cog.warp_tifs(dsts, local_dir)
        merged_tif = cog.merge_tifs(warped_paths, local_dir)
        cog.add_overviews(merged_tif)
        cog_path = cog.convert_to_cog(merged_tif, local_dir)
        if same_path:
            updated_scene = upload_tif(
                cog_path, scene,
                os.path.join('user-uploads', scene.owner, '{}_COG.tif'.format(scene.id)),
                os.path.join('user-uploads', urllib.quote_plus(scene.owner), '{}_COG.tif'.format(scene.id))
            )
        else:
            updated_scene = upload_tif(cog_path, scene)
        os.remove(cog_path)
        return updated_scene
Exemplo n.º 3
0
    def generate_scenes(self):
        """Create a Scene and associated Image for each GeoTiff in self.s3_path
        Returns:
            Generator of Scenes
        """
        s3 = boto3.resource('s3')
        for infile in self.files:
            # We can't use the temp file as a context manager because it'll be opened/closed multiple
            # times and by default is deleted when it's closed. So we use try/finally to ensure that
            # it gets cleaned up.
            bucket_name, key = s3_bucket_and_key_from_url(infile)
            filename = os.path.basename(key)
            logger.info('Downloading %s => %s', infile, filename)
            bucket = s3.Bucket(bucket_name)
            with get_tempdir() as tempdir:
                tmp_fname = os.path.join(tempdir, filename)
                bucket.download_file(key, tmp_fname)
                cog.add_overviews(tmp_fname)
                cog_path = cog.convert_to_cog(tmp_fname, tempdir)
                scene = self.create_geotiff_scene(
                    tmp_fname,
                    os.path.splitext(filename)[0])
                scene.ingestLocation = upload_tifs([cog_path], self.owner,
                                                   scene.id)[0]
                images = [
                    self.create_geotiff_image(
                        tmp_fname, urllib.unquote(scene.ingestLocation), scene,
                        cog_path)
                ]

            scene.thumbnails = []
            scene.images = images
            yield scene
Exemplo n.º 4
0
def create_cog(image_locations, scene):
    with get_tempdir() as local_dir:
        dsts = [os.path.join(local_dir, fname) for _, fname in image_locations]
        cog.fetch_imagery(image_locations, local_dir)
        warped_paths = cog.warp_tifs(dsts, local_dir)
        merged_tif = cog.merge_tifs(warped_paths, local_dir)
        cog.add_overviews(merged_tif)
        cog_path = cog.convert_to_cog(merged_tif, local_dir)
        updated_scene = upload_tif(cog_path, scene)
        updated_scene.update()
Exemplo n.º 5
0
    def copy_asset_to_s3(self, prefix, planet_id):
        """Make the Planet tif available to Rater Foundry

        This function downloads the basic analytic tif from planet,
        then reuploads it to s3 in a directory for planet files.

        Args:
            planet_id (str): id of the planet image to download

        Returns:
            dict: geojson for the overview of the planet tif
        """

        split_id = planet_id.split(':')
        item_type = split_id[0]
        item_id = split_id[1]
        if len(split_id) == 2:
            asset_type = None
        elif len(split_id) == 3:
            asset_type = split_id[2]
        logger.info('Retrieving item type %s with id %s', item_type, item_id)
        item = self.get_item(item_id, item_type)
        item_id = item['id']

        assets = self.get_assets_by_id(item_id, item_type)
        asset_type = PlanetSceneFactory.get_asset_type(assets, asset_type)
        updated_assets = self.activate_asset_and_wait(asset_type, assets,
                                                      item_id, item_type)

        temp_tif_file = self.download_planet_tif(prefix, asset_type,
                                                 updated_assets, item_id)
        full_path_to_temp_tif = os.path.join(prefix, temp_tif_file)
        cog.add_overviews(full_path_to_temp_tif)
        cog_path = cog.convert_to_cog(full_path_to_temp_tif, prefix)
        bucket, s3_path = self.upload_planet_tif(asset_type, item_id,
                                                 item_type, cog_path)

        analytic_xml = self.get_analytic_xml(assets, item_id, item_type)
        reflectance_coefficients = PlanetSceneFactory.get_reflectance_coefficients(
            analytic_xml)
        item['properties'].update(reflectance_coefficients)

        item['added_props'] = {}
        item['added_props']['localPath'] = cog_path
        item['added_props']['s3Location'] = 's3://{}/{}'.format(
            bucket, s3_path)
        item['added_props']['asset_type'] = asset_type

        # Return the json representation of the item
        return item, temp_tif_file
Exemplo n.º 6
0
def process_to_cog(prefix, gcs_prefix, landsat_id, config):
    logger.info('Fetching all bands')
    for band in config.bands.keys():
        fetch_band(prefix, gcs_prefix, band, landsat_id)
    cog_fname = '{}_COG.tif'.format(landsat_id)
    stacked_fname = '{}_STACKED.tif'.format(landsat_id)
    filenames = {
        'COG': os.path.join(prefix, cog_fname),
        'STACKED': os.path.join(prefix, stacked_fname)
    }
    local_paths = sorted(glob.glob('/{}/{}*.TIF'.format(prefix, landsat_id)))
    warped_paths = cog.warp_tifs(local_paths, prefix)
    merged = cog.merge_tifs(warped_paths, prefix)
    cog.add_overviews(merged)
    cog_path = cog.convert_to_cog(merged, prefix)
    shutil.move(cog_path, filenames['COG'])
    return (filenames['COG'], cog_fname)
Exemplo n.º 7
0
def process_to_cog(prefix, gcs_prefix, landsat_id, config):
    logger.info('Fetching all bands')
    for band in config.bands.keys():
        fetch_band(prefix, gcs_prefix, band, landsat_id)
    cog_fname = '{}_COG.tif'.format(landsat_id)
    stacked_fname = '{}_STACKED.tif'.format(landsat_id)
    filenames = {
        'COG': os.path.join(prefix, cog_fname),
        'STACKED': os.path.join(prefix, stacked_fname)
    }
    local_paths = sorted(glob.glob('{}/{}*.TIF'.format(prefix, landsat_id)))
    warped_paths = cog.warp_tifs(local_paths, prefix)
    merged = cog.merge_tifs(warped_paths, prefix)
    cog.add_overviews(merged)
    cog_path = cog.convert_to_cog(merged, prefix)
    shutil.move(cog_path, filenames['COG'])
    return (filenames['COG'], cog_fname)
Exemplo n.º 8
0
    def copy_asset_to_s3(self, prefix, planet_id):
        """Make the Planet tif available to Rater Foundry

        This function downloads the basic analytic tif from planet,
        then reuploads it to s3 in a directory for planet files.

        Args:
            planet_id (str): id of the planet image to download

        Returns:
            dict: geojson for the overview of the planet tif
        """

        split_id = planet_id.split(':')
        item_type = split_id[0]
        item_id = split_id[1]
        if len(split_id) == 2:
            asset_type = None
        elif len(split_id) == 3:
            asset_type = split_id[2]
        logger.info('Retrieving item type %s with id %s', item_type, item_id)
        item = self.get_item(item_id, item_type)
        item_id = item['id']

        assets = self.get_assets_by_id(item_id, item_type)
        asset_type = PlanetSceneFactory.get_asset_type(assets, asset_type)
        updated_assets = self.activate_asset_and_wait(asset_type, assets, item_id, item_type)

        temp_tif_file = self.download_planet_tif(prefix, asset_type, updated_assets, item_id)
        full_path_to_temp_tif = os.path.join(prefix, temp_tif_file)
        cog.add_overviews(full_path_to_temp_tif)
        cog_path = cog.convert_to_cog(full_path_to_temp_tif, prefix)
        bucket, s3_path = self.upload_planet_tif(asset_type, item_id, item_type, cog_path)

        analytic_xml = self.get_analytic_xml(assets, item_id, item_type)
        reflectance_coefficients = PlanetSceneFactory.get_reflectance_coefficients(analytic_xml)
        item['properties'].update(reflectance_coefficients)

        item['added_props'] = {}
        item['added_props']['localPath'] = cog_path
        item['added_props']['s3Location'] = 's3://{}/{}'.format(bucket, s3_path)
        item['added_props']['asset_type'] = asset_type

        # Return the json representation of the item
        return item, temp_tif_file
Exemplo n.º 9
0
def create_cog(image_locations, scene, same_path=False):
    with get_tempdir() as local_dir:
        dsts = [os.path.join(local_dir, fname) for _, fname in image_locations]
        cog.fetch_imagery(image_locations, local_dir)
        warped_paths = cog.warp_tifs(dsts, local_dir)
        merged_tif = cog.merge_tifs(warped_paths, local_dir)
        cog.add_overviews(merged_tif)
        cog_path = cog.convert_to_cog(merged_tif, local_dir)
        if same_path:
            updated_scene = upload_tif(
                cog_path, scene,
                os.path.join('user-uploads', scene.owner, '{}_COG.tif'.format(scene.id)),
                os.path.join('user-uploads', urllib.quote_plus(scene.owner), '{}_COG.tif'.format(scene.id))
            )
        else:
            updated_scene = upload_tif(cog_path, scene)
        updated_scene.update()
        os.remove(cog_path)
Exemplo n.º 10
0
def create_cog(image_locations, scene, same_path=False):
    with get_tempdir() as local_dir:
        dsts = [os.path.join(local_dir, fname) for _, fname in image_locations]
        cog.fetch_imagery(image_locations, local_dir)
        warped_paths = cog.warp_tifs(dsts, local_dir)
        merged_tif = cog.merge_tifs(warped_paths, local_dir)
        cog.add_overviews(merged_tif)
        cog_path = cog.convert_to_cog(merged_tif, local_dir)
        if same_path:
            updated_scene = upload_tif(
                cog_path, scene,
                os.path.join('user-uploads', scene.owner,
                             '{}_COG.tif'.format(scene.id)),
                os.path.join('user-uploads', urllib.quote_plus(scene.owner),
                             '{}_COG.tif'.format(scene.id)))
        else:
            updated_scene = upload_tif(cog_path, scene)
        updated_scene.update()
        os.remove(cog_path)
Exemplo n.º 11
0
def create_geotiffs(modis_path, local_dir):
    """Create geotiffs from MODIS HDF

    1. Create separate tifs for each band
    2. Combine tifs into a single tif
    3. Warp tif to web mercator (for quicker access)
    4. Generate COG

    Args:
        modis_path (str): path to modis HDF file
        local_dir (str): directory to output tiffs to
    """
    logger.info('Preparing to create geotiffs')

    post_web_mercator_path = os.path.join(local_dir, 'warp2.tif')

    tifs = sorted(hdf_to_geotiffs(modis_path, local_dir))
    logger.info('Tifs: %s', '\n'.join(tifs))
    warped_paths = cog.warp_tifs(tifs, local_dir)
    merged_tif = cog.merge_tifs(warped_paths, local_dir)
    warp_tif(merged_tif, post_web_mercator_path)
    cog.add_overviews(post_web_mercator_path)
    cog_path = cog.convert_to_cog(post_web_mercator_path, local_dir)
    return [cog_path]
Exemplo n.º 12
0
def create_geotiffs(modis_path, local_dir):
    """Create geotiffs from MODIS HDF

    1. Create separate tifs for each band
    2. Combine tifs into a single tif
    3. Warp tif to web mercator (for quicker access)
    4. Generate COG

    Args:
        modis_path (str): path to modis HDF file
        local_dir (str): directory to output tiffs to
    """
    logger.info('Preparing to create geotiffs')

    post_web_mercator_path = os.path.join(local_dir, 'warp2.tif')

    tifs = sorted(hdf_to_geotiffs(modis_path, local_dir))
    logger.info('Tifs: %s', '\n'.join(tifs))
    warped_paths = cog.warp_tifs(tifs, local_dir)
    merged_tif = cog.merge_tifs(warped_paths, local_dir)
    warp_tif(merged_tif, post_web_mercator_path)
    cog.add_overviews(post_web_mercator_path)
    cog_path = cog.convert_to_cog(post_web_mercator_path, local_dir)
    return [cog_path]