def make_permanent(self):
"""Move the tile file to its permanent location."""
source_dir = os.path.abspath(os.path.dirname(self.temp_tile_output_path))
dest_dir = os.path.abspath(os.path.dirname(self.tile_output_path))
create_directory(dest_dir)
# If required, edit paths in re-written .vrt file and move .nc file
if self.nc_tile_output_path:
vrt_file = open(self.temp_tile_output_path, 'r')
vrt_string = vrt_file.read()
vrt_file.close()
vrt_string = vrt_string.replace(source_dir, dest_dir) # Update all paths in VRT file
vrt_file = open(self.tile_output_path, 'w')
vrt_file.write(vrt_string)
vrt_file.close()
# Move .nc file
shutil.move(self.nc_temp_tile_output_path, self.nc_tile_output_path)
else: # No .vrt file required - just move the tile file
shutil.move(self.temp_tile_output_path, self.tile_output_path)
def buildvrt(self, temp_dir):
"""Given a dataset_record and corresponding dataset, build the vrt that
will be used to reproject the dataset's data to tile coordinates"""
# Make the list of filenames from the dataset_path/scene01 and each
# file_number's file_pattern. Also get list of nodata_value.
self.source_file_list, self.nodata_list = self.list_source_files()
nodata_value = self.nodata_list[0]
# TODO: check that this works for PQA where nodata_value is None
if nodata_value is not None:
nodata_spec = ["-srcnodata", "%d" % nodata_value, "-vrtnodata", "%d" % (nodata_value)]
else:
nodata_spec = []
# Form the vrt_band_stack_filename.
# This is done using
# args = shlex.split(command_line)
# where command_line is the buildvrt command
create_directory(temp_dir)
self.vrt_name = self.get_vrt_name(temp_dir)
# Build the vrt
buildvrt_cmd = ["gdalbuildvrt", "-separate", "-q"]
buildvrt_cmd.extend(nodata_spec)
buildvrt_cmd.extend(["-overwrite", "%s" % self.vrt_name])
buildvrt_cmd.extend(self.source_file_list)
# for fle in self.source_file_list:
# buildvrt_cmd.append(fle)
# buildvrt_cmd = ' '.join(buildvrt_cmd)
result = execute(buildvrt_cmd, shell=False)
if result["returncode"] != 0:
raise DatasetError(
"Unable to perform gdalbuildvrt: " + '"%s" failed: %s' % (buildvrt_cmd, result["stderr"])
)
# Add the metadata and return the band_stack as a gdal datatset, storing
# as an attribute of the Bandstack object
self.vrt_band_stack = self.add_metadata(self.vrt_name)
def __init__(self, datacube):
"""Initialise the collection object."""
self.datacube = datacube
self.db = IngestDBWrapper(datacube.db_connection)
self.new_bands = self.__reindex_bands(datacube.bands)
self.transaction_stack = []
self.temp_tile_directory = os.path.join(self.datacube.tile_root,
'ingest_temp',
self.datacube.process_id)
create_directory(self.temp_tile_directory)
def __init__(self, datacube):
"""Initialise the collection object."""
self.datacube = datacube
self.db = IngestDBWrapper(datacube.db_connection)
self.new_bands = self.__reindex_bands(datacube.bands)
self.transaction_stack = []
self.temp_tile_directory = os.path.join(self.datacube.tile_root,
'ingest_temp',
self.datacube.process_id)
create_directory(self.temp_tile_directory)
def buildvrt(self, temp_dir):
"""Given a dataset_record and corresponding dataset, build the vrt that
will be used to reproject the dataset's data to tile coordinates"""
#Make the list of filenames from the dataset_path/scene01 and each
#file_number's file_pattern. Also get list of nodata_value.
self.source_file_list, self.nodata_list = self.list_source_files()
nodata_value = self.nodata_list[0]
#TODO: check that this works for PQA where nodata_value is None
if nodata_value is not None:
nodata_spec = [
"-srcnodata",
"%d" % nodata_value, "-vrtnodata",
"%d" % (nodata_value)
]
else:
nodata_spec = []
#Form the vrt_band_stack_filename.
#This is done using
#args = shlex.split(command_line)
#where command_line is the buildvrt command
create_directory(temp_dir)
self.vrt_name = self.get_vrt_name(temp_dir)
#Build the vrt
buildvrt_cmd = [
"gdalbuildvrt",
"-separate",
"-q",
]
buildvrt_cmd.extend(nodata_spec)
buildvrt_cmd.extend(["-overwrite", "%s" % self.vrt_name])
buildvrt_cmd.extend(self.source_file_list)
#for fle in self.source_file_list:
# buildvrt_cmd.append(fle)
#buildvrt_cmd = ' '.join(buildvrt_cmd)
result = execute(buildvrt_cmd, shell=False)
if result['returncode'] != 0:
raise DatasetError('Unable to perform gdalbuildvrt: ' +
'"%s" failed: %s'\
% (buildvrt_cmd, result['stderr']))
#Add the metadata and return the band_stack as a gdal datatset, storing
#as an attribute of the Bandstack object
self.vrt_band_stack = self.add_metadata(self.vrt_name)
def __get_mosaic_paths(tile_pathname, extension, temp_tile_dir):
"""Generate the temporary and final pathnames for the mosaic.
'tile_pathname' is the path to the first tile in the mosaic.
'extension' is the extension to use for the mosaic filename.
Returns a tuple (mosaic_temp_path, mosaic_final_path).
"""
(tile_dir, tile_basename) = os.path.split(tile_pathname)
mosaic_final_dir = os.path.join(tile_dir, 'mosaic_cache')
create_directory(mosaic_final_dir)
mosaic_temp_dir = os.path.join(temp_tile_dir, 'mosaic_cache')
create_directory(mosaic_temp_dir)
mosaic_basename = re.sub(r'\.\w+$', extension, tile_basename)
mosaic_temp_path = os.path.join(mosaic_temp_dir, mosaic_basename)
mosaic_final_path = os.path.join(mosaic_final_dir, mosaic_basename)
return (mosaic_temp_path, mosaic_final_path)
def test_create_multi_complex(self):
"""Create a multi level directory, complex test."""
cube_util.create_directory(self.multi_dir_top_path)
self.assertTrue(os.path.isdir(self.multi_dir_top_path),
"Directory %s not created." % self.multi_dir_top_path)
cube_util.create_directory(self.multi_dir_full_path)
self.assertTrue(os.path.isdir(self.multi_dir_full_path),
"Directory %s not created." % self.multi_dir_full_path)
cube_util.create_directory(self.multi_dir_full_path)
def test_create_multi_complex(self):
"""Create a multi level directory, complex test."""
cube_util.create_directory(self.multi_dir_top_path)
self.assertTrue(os.path.isdir(self.multi_dir_top_path),
"Directory %s not created." % self.multi_dir_top_path)
cube_util.create_directory(self.multi_dir_full_path)
self.assertTrue(os.path.isdir(self.multi_dir_full_path),
"Directory %s not created." % self.multi_dir_full_path)
cube_util.create_directory(self.multi_dir_full_path)
def test_create_one(self):
"""Create a single level directory."""
cube_util.create_directory(self.single_dir_path)
self.assertTrue(os.path.isdir(self.single_dir_path),
"Directory %s not created." % self.single_dir_path)
def make_permanent(self):
"""Move the tile file to its permanent location."""
create_directory(os.path.dirname(self.tile_output_path))
shutil.move(self.temp_tile_output_path, self.tile_output_path)
def make_permanent(self):
"""Move the tile file to its permanent location."""
create_directory(os.path.dirname(self.tile_output_path))
shutil.move(self.temp_tile_output_path, self.tile_output_path)
def test_create_one(self):
"""Create a single level directory."""
cube_util.create_directory(self.single_dir_path)
self.assertTrue(os.path.isdir(self.single_dir_path),
"Directory %s not created." % self.single_dir_path)