def unpack_wv_to_folder(zip_path, unpack_folder):
with portalocker.Lock(zip_path, 'r', timeout=300) as unused: #pylint: disable=W0612
# Check if we already unpacked this data
(tif_path, imd_path) = get_files_from_unpack_folder(unpack_folder)
if imd_path and tif_path:
pass
else:
print('Unpacking file ' + zip_path + ' to folder ' + unpack_folder)
utilities.unpack_to_folder(zip_path, unpack_folder)
# some worldview zip files have a subdirectory with the name of the image
if not os.path.exists(
os.path.join(unpack_folder, 'vendor_metadata')):
subdir = os.path.join(
unpack_folder,
os.path.splitext(os.path.basename(zip_path))[0])
if not os.path.exists(os.path.join(subdir, 'vendor_metadata')):
raise Exception('vendor_metadata not found in %s.' %
(zip_path))
for filename in os.listdir(subdir):
os.rename(os.path.join(subdir, filename),
os.path.join(unpack_folder, filename))
os.rmdir(subdir)
(tif_path, imd_path) = get_files_from_unpack_folder(unpack_folder)
return (tif_path, imd_path)
def _unpack(self, paths):
# Get the folder where this will be stored from the cache manager
unpack_folder = config.io.cache.manager().register_item(self._name)
with portalocker.Lock(paths, 'r', timeout=300) as unused: #pylint: disable=W0612
# Check if we already unpacked this data
(tif_path, imd_path) = _get_files_from_unpack_folder(unpack_folder)
if imd_path and tif_path:
#tf.print('Already have unpacked files in ' + unpack_folder,
# output_stream=sys.stdout)
pass
else:
tf.print('Unpacking file ' + paths + ' to folder ' + unpack_folder,
output_stream=sys.stdout)
utilities.unpack_to_folder(paths, unpack_folder)
# some worldview zip files have a subdirectory with the name of the image
if not os.path.exists(os.path.join(unpack_folder, 'vendor_metadata')):
subdir = os.path.join(unpack_folder, os.path.splitext(os.path.basename(paths))[0])
if not os.path.exists(os.path.join(subdir, 'vendor_metadata')):
raise Exception('vendor_metadata not found in %s.' % (paths))
for filename in os.listdir(subdir):
os.rename(os.path.join(subdir, filename), os.path.join(unpack_folder, filename))
os.rmdir(subdir)
(tif_path, imd_path) = _get_files_from_unpack_folder(unpack_folder)
return (tif_path, imd_path)
def _prep(self, paths):
"""Prepares a Landsat file from the archive for processing.
Returns [band, paths, in, order, ...]
Uses the bands specified in _get_landsat_bands_to_use()
TODO: Handle bands which are not 30 meters!
TODO: Apply TOA conversion!
"""
scene_info = get_scene_info(paths)
self._sensor = scene_info['sensor']
self._lpath = scene_info['lpath']
self._lrow = scene_info['lrow']
self._date = scene_info['date']
# Get the folder where this will be stored from the cache manager
name = '_'.join([self._sensor, self._lpath, self._lrow, self._date])
untar_folder = config.io.cache.manager().register_item(name)
# Check if we already unpacked this data
all_files_present = False
if os.path.exists(untar_folder):
mtl_path = _find_mtl_file(untar_folder)
if mtl_path:
mtl_data = _parse_mtl_file(mtl_path)
all_files_present = _check_if_files_present(
mtl_data, untar_folder)
if all_files_present:
print('Already have unpacked files in ' + untar_folder)
else:
print('Unpacking tar file ' + paths + ' to folder ' + untar_folder)
utilities.unpack_to_folder(paths, untar_folder)
bands_to_use = _get_landsat_bands_to_use(
self._sensor) if self._bands is None else self._bands
# Generate all the band file names (the MTL file is not returned)
self._mtl_path = _find_mtl_file(untar_folder)
self._mtl_data = _parse_mtl_file(self._mtl_path)
output_paths = _get_band_paths(self._mtl_data, untar_folder,
bands_to_use)
# Check that the files exist
for p in output_paths:
if not os.path.exists(p):
raise Exception('Did not find expected file: ' + p +
' after unpacking tar file ' + paths)
return output_paths
def _unpack(self, paths):
# Get the folder where this will be stored from the cache manager
name = '_'.join([self._sensor, self._date])
unpack_folder = config.io.cache.manager().register_item(name)
# Check if we already unpacked this data
(tif_path, imd_path) = _get_files_from_unpack_folder(unpack_folder)
if imd_path and tif_path:
#print('Already have unpacked files in ' + unpack_folder)
pass
else:
print('Unpacking file ' + paths + ' to folder ' + unpack_folder)
utilities.unpack_to_folder(paths, unpack_folder)
(tif_path, imd_path) = _get_files_from_unpack_folder(unpack_folder)
return (tif_path, imd_path)
def unpack_inputs(tar_folder, unpack_folder):
"""Make sure all of the input label files are untarred.
The unpack folder can be the same as the tar folder.
Returns the list of label files.
"""
if not os.path.exists(unpack_folder):
os.mkdir(unpack_folder)
file_list = []
# Loop through tar files
input_list = os.listdir(tar_folder)
for f in input_list:
ext = os.path.splitext(f)[1]
if ext != '.tar':
continue
# The name of the input tar does not fully match the untar file names
name = os.path.basename(f)
parts = name.split('_')
prefix = '_'.join(parts[0:4])
# Look to see if we have a matching label file
tar_path = os.path.join(tar_folder, f)
label_path = look_for_file(unpack_folder, [prefix, '_INWM.tif'])
# If we did not find the INWM file, untar.
if not label_path:
utilities.unpack_to_folder(tar_path, unpack_folder)
# Look again for a matching INWM file
label_path = look_for_file(unpack_folder, [prefix, '_INWM.tif'])
if not label_path:
raise Exception('Failed to untar label file: ' + tar_path)
file_list.append(label_path)
return file_list
def unpack_s1_to_folder(zip_path, unpack_folder):
'''Returns the merged image path from the unpack folder.
Unpacks the zip file and merges the source images as needed.'''
if shutil.which('gpt') is None:
raise Exception('Could not find the tool "gpt", do you have ESA SNAP installed and on your PATH?')
with portalocker.Lock(zip_path, 'r', timeout=300) as unused: #pylint: disable=W0612
merged_path = get_merged_path(unpack_folder)
try:
test_image = tiff.TiffImage(merged_path) #pylint: disable=W0612
except Exception: #pylint: disable=W0703
test_image = None
if test_image: # Merged image is ready to use
print('Already have unpacked files in ' + unpack_folder)
return merged_path
# Otherwise go through the entire unpack process
NUM_SOURCE_CHANNELS = 2
need_to_unpack = True
if os.path.exists(unpack_folder):
source_image_paths = get_files_from_unpack_folder(unpack_folder)
if len(source_image_paths) == NUM_SOURCE_CHANNELS:
need_to_unpack = False
print('Already have files')
else:
print('Clearing unpack folder missing image files.')
os.system('rm -rf ' + unpack_folder)
if need_to_unpack:
print('Unpacking file ' + zip_path + ' to folder ' + unpack_folder)
utilities.unpack_to_folder(zip_path, unpack_folder)
subdirs = os.listdir(unpack_folder)
if len(subdirs) != 1:
raise Exception('Unexpected Sentinel1 subdirectories: ' + str(subdirs))
cmd = 'mv ' + os.path.join(unpack_folder, subdirs[0]) +'/* ' + unpack_folder
print(cmd)
os.system(cmd)
source_image_paths = get_files_from_unpack_folder(unpack_folder)
if len(source_image_paths) != NUM_SOURCE_CHANNELS:
raise Exception('Did not find two image files in ' + zip_path)
USE_SNAP = True # To get real results we need to use SNAP
if USE_SNAP: # Requires the Sentinel1 processing software to be installed
# Run the preconfigured SNAP preprocessing graph
# - The SNAP tool *must* write to a .tif extension, so we have to
# rename the file if we want something else.
temp_out_path = merged_path.replace('.vrt', '.tif')
run_ffilipponi_preprocessing(unpack_folder, temp_out_path)
dimap_path = temp_out_path + '.dim'
cmd = 'pconvert -s 0,0 -f GeoTIFF-BigTiff -o ' + os.path.dirname(temp_out_path) +' '+ dimap_path
print(cmd)
os.system(cmd)
MIN_IMAGE_SIZE = 1024*1024*500 # 500 MB, expected size is much larger
if not os.path.exists(temp_out_path):
raise Exception('Failed to run ESA SNAP preprocessing!')
if os.path.getsize(temp_out_path) < MIN_IMAGE_SIZE:
raise Exception('SNAP encountered a problem processing the file!')
os.system('mv ' + temp_out_path + ' ' + merged_path)
else:
# Generate a merged file containing all input images as an N channel image
cmd = 'gdalbuildvrt -separate ' + merged_path
for f in source_image_paths:
cmd += ' ' + f
print(cmd)
os.system(cmd)
# Verify that we generated a valid image file
try:
test_image = tiff.TiffImage(merged_path) #pylint: disable=W0612
except Exception as e: #pylint: disable=W0703
raise Exception('Failed to generate merged Sentinel1 file: ' + merged_path) from e
return merged_path