Exemple #1
0
    def make_permanent(self):
        """Move the tile file to its permanent location."""

        source_dir = os.path.abspath(os.path.dirname(self.temp_tile_output_path))
        dest_dir = os.path.abspath(os.path.dirname(self.tile_output_path))
        
        create_directory(dest_dir)
        
        # If required, edit paths in re-written .vrt file and move .nc file 
        if self.nc_tile_output_path:
            vrt_file = open(self.temp_tile_output_path, 'r')
            vrt_string = vrt_file.read()
            vrt_file.close()
            
            vrt_string = vrt_string.replace(source_dir, dest_dir) # Update all paths in VRT file
            
            vrt_file = open(self.tile_output_path, 'w')
            vrt_file.write(vrt_string)
            vrt_file.close()
            
            # Move .nc file
            shutil.move(self.nc_temp_tile_output_path, self.nc_tile_output_path)

        else: # No .vrt file required - just move the tile file
            shutil.move(self.temp_tile_output_path, self.tile_output_path)
    def buildvrt(self, temp_dir):
        """Given a dataset_record and corresponding dataset, build the vrt that
        will be used to reproject the dataset's data to tile coordinates"""

        # Make the list of filenames from the dataset_path/scene01 and each
        # file_number's file_pattern. Also get list of nodata_value.
        self.source_file_list, self.nodata_list = self.list_source_files()
        nodata_value = self.nodata_list[0]
        # TODO: check that this works for PQA where nodata_value is None
        if nodata_value is not None:
            nodata_spec = ["-srcnodata", "%d" % nodata_value, "-vrtnodata", "%d" % (nodata_value)]
        else:
            nodata_spec = []
        # Form the vrt_band_stack_filename.
        # This is done using
        # args = shlex.split(command_line)
        # where command_line is the buildvrt command
        create_directory(temp_dir)
        self.vrt_name = self.get_vrt_name(temp_dir)
        # Build the vrt
        buildvrt_cmd = ["gdalbuildvrt", "-separate", "-q"]
        buildvrt_cmd.extend(nodata_spec)
        buildvrt_cmd.extend(["-overwrite", "%s" % self.vrt_name])
        buildvrt_cmd.extend(self.source_file_list)
        # for fle in self.source_file_list:
        #    buildvrt_cmd.append(fle)
        # buildvrt_cmd = ' '.join(buildvrt_cmd)
        result = execute(buildvrt_cmd, shell=False)
        if result["returncode"] != 0:
            raise DatasetError(
                "Unable to perform gdalbuildvrt: " + '"%s" failed: %s' % (buildvrt_cmd, result["stderr"])
            )
        # Add the metadata and return the band_stack as a gdal datatset, storing
        # as an attribute of the Bandstack object
        self.vrt_band_stack = self.add_metadata(self.vrt_name)
    def __init__(self, datacube):
        """Initialise the collection object."""

        self.datacube = datacube
        self.db = IngestDBWrapper(datacube.db_connection)
        self.new_bands = self.__reindex_bands(datacube.bands)
        self.transaction_stack = []

        self.temp_tile_directory = os.path.join(self.datacube.tile_root,
                                                'ingest_temp',
                                                self.datacube.process_id)
        create_directory(self.temp_tile_directory)
Exemple #4
0
    def __init__(self, datacube):
        """Initialise the collection object."""

        self.datacube = datacube
        self.db = IngestDBWrapper(datacube.db_connection)
        self.new_bands = self.__reindex_bands(datacube.bands)
        self.transaction_stack = []

        self.temp_tile_directory = os.path.join(self.datacube.tile_root,
                                                'ingest_temp',
                                                self.datacube.process_id)
        create_directory(self.temp_tile_directory)
Exemple #5
0
    def buildvrt(self, temp_dir):
        """Given a dataset_record and corresponding dataset, build the vrt that
        will be used to reproject the dataset's data to tile coordinates"""

        #Make the list of filenames from the dataset_path/scene01 and each
        #file_number's file_pattern. Also get list of nodata_value.
        self.source_file_list, self.nodata_list = self.list_source_files()
        nodata_value = self.nodata_list[0]
        #TODO: check that this works for PQA where nodata_value is None
        if nodata_value is not None:
            nodata_spec = [
                "-srcnodata",
                "%d" % nodata_value, "-vrtnodata",
                "%d" % (nodata_value)
            ]
        else:
            nodata_spec = []
        #Form the vrt_band_stack_filename.
        #This is done using
        #args = shlex.split(command_line)
        #where command_line is the buildvrt command
        create_directory(temp_dir)
        self.vrt_name = self.get_vrt_name(temp_dir)
        #Build the vrt
        buildvrt_cmd = [
            "gdalbuildvrt",
            "-separate",
            "-q",
        ]
        buildvrt_cmd.extend(nodata_spec)
        buildvrt_cmd.extend(["-overwrite", "%s" % self.vrt_name])
        buildvrt_cmd.extend(self.source_file_list)
        #for fle in self.source_file_list:
        #    buildvrt_cmd.append(fle)
        #buildvrt_cmd = ' '.join(buildvrt_cmd)
        result = execute(buildvrt_cmd, shell=False)
        if result['returncode'] != 0:
            raise DatasetError('Unable to perform gdalbuildvrt: ' +
                               '"%s" failed: %s'\
                                   % (buildvrt_cmd, result['stderr']))
        #Add the metadata and return the band_stack as a gdal datatset, storing
        #as an attribute of the Bandstack object
        self.vrt_band_stack = self.add_metadata(self.vrt_name)
Exemple #6
0
    def __get_mosaic_paths(tile_pathname, extension, temp_tile_dir):
        """Generate the temporary and final pathnames for the mosaic.

        'tile_pathname' is the path to the first tile in the mosaic.
        'extension' is the extension to use for the mosaic filename.
        Returns a tuple (mosaic_temp_path, mosaic_final_path).
        """

        (tile_dir, tile_basename) = os.path.split(tile_pathname)

        mosaic_final_dir = os.path.join(tile_dir, 'mosaic_cache')
        create_directory(mosaic_final_dir)

        mosaic_temp_dir = os.path.join(temp_tile_dir, 'mosaic_cache')
        create_directory(mosaic_temp_dir)

        mosaic_basename = re.sub(r'\.\w+$', extension, tile_basename)

        mosaic_temp_path = os.path.join(mosaic_temp_dir, mosaic_basename)
        mosaic_final_path = os.path.join(mosaic_final_dir, mosaic_basename)

        return (mosaic_temp_path, mosaic_final_path)
Exemple #7
0
    def test_create_multi_complex(self):
        """Create a multi level directory, complex test."""

        cube_util.create_directory(self.multi_dir_top_path)
        self.assertTrue(os.path.isdir(self.multi_dir_top_path),
                        "Directory %s not created." % self.multi_dir_top_path)

        cube_util.create_directory(self.multi_dir_full_path)
        self.assertTrue(os.path.isdir(self.multi_dir_full_path),
                        "Directory %s not created." % self.multi_dir_full_path)

        cube_util.create_directory(self.multi_dir_full_path)
    def test_create_multi_complex(self):
        """Create a multi level directory, complex test."""

        cube_util.create_directory(self.multi_dir_top_path)
        self.assertTrue(os.path.isdir(self.multi_dir_top_path),
                        "Directory %s not created." % self.multi_dir_top_path)

        cube_util.create_directory(self.multi_dir_full_path)
        self.assertTrue(os.path.isdir(self.multi_dir_full_path),
                        "Directory %s not created." % self.multi_dir_full_path)

        cube_util.create_directory(self.multi_dir_full_path)
Exemple #9
0
    def test_create_one(self):
        """Create a single level directory."""

        cube_util.create_directory(self.single_dir_path)
        self.assertTrue(os.path.isdir(self.single_dir_path),
                        "Directory %s not created." % self.single_dir_path)
Exemple #10
0
    def make_permanent(self):
        """Move the tile file to its permanent location."""

        create_directory(os.path.dirname(self.tile_output_path))
        shutil.move(self.temp_tile_output_path, self.tile_output_path)
Exemple #11
0
    def make_permanent(self):
        """Move the tile file to its permanent location."""

        create_directory(os.path.dirname(self.tile_output_path))
        shutil.move(self.temp_tile_output_path, self.tile_output_path)
    def test_create_one(self):
        """Create a single level directory."""

        cube_util.create_directory(self.single_dir_path)
        self.assertTrue(os.path.isdir(self.single_dir_path),
                        "Directory %s not created." % self.single_dir_path)