def unpack_wv_to_folder(zip_path, unpack_folder): with portalocker.Lock(zip_path, 'r', timeout=300) as unused: #pylint: disable=W0612 # Check if we already unpacked this data (tif_path, imd_path) = get_files_from_unpack_folder(unpack_folder) if imd_path and tif_path: pass else: print('Unpacking file ' + zip_path + ' to folder ' + unpack_folder) utilities.unpack_to_folder(zip_path, unpack_folder) # some worldview zip files have a subdirectory with the name of the image if not os.path.exists( os.path.join(unpack_folder, 'vendor_metadata')): subdir = os.path.join( unpack_folder, os.path.splitext(os.path.basename(zip_path))[0]) if not os.path.exists(os.path.join(subdir, 'vendor_metadata')): raise Exception('vendor_metadata not found in %s.' % (zip_path)) for filename in os.listdir(subdir): os.rename(os.path.join(subdir, filename), os.path.join(unpack_folder, filename)) os.rmdir(subdir) (tif_path, imd_path) = get_files_from_unpack_folder(unpack_folder) return (tif_path, imd_path)
def _unpack(self, paths): # Get the folder where this will be stored from the cache manager unpack_folder = config.io.cache.manager().register_item(self._name) with portalocker.Lock(paths, 'r', timeout=300) as unused: #pylint: disable=W0612 # Check if we already unpacked this data (tif_path, imd_path) = _get_files_from_unpack_folder(unpack_folder) if imd_path and tif_path: #tf.print('Already have unpacked files in ' + unpack_folder, # output_stream=sys.stdout) pass else: tf.print('Unpacking file ' + paths + ' to folder ' + unpack_folder, output_stream=sys.stdout) utilities.unpack_to_folder(paths, unpack_folder) # some worldview zip files have a subdirectory with the name of the image if not os.path.exists(os.path.join(unpack_folder, 'vendor_metadata')): subdir = os.path.join(unpack_folder, os.path.splitext(os.path.basename(paths))[0]) if not os.path.exists(os.path.join(subdir, 'vendor_metadata')): raise Exception('vendor_metadata not found in %s.' % (paths)) for filename in os.listdir(subdir): os.rename(os.path.join(subdir, filename), os.path.join(unpack_folder, filename)) os.rmdir(subdir) (tif_path, imd_path) = _get_files_from_unpack_folder(unpack_folder) return (tif_path, imd_path)
def _prep(self, paths): """Prepares a Landsat file from the archive for processing. Returns [band, paths, in, order, ...] Uses the bands specified in _get_landsat_bands_to_use() TODO: Handle bands which are not 30 meters! TODO: Apply TOA conversion! """ scene_info = get_scene_info(paths) self._sensor = scene_info['sensor'] self._lpath = scene_info['lpath'] self._lrow = scene_info['lrow'] self._date = scene_info['date'] # Get the folder where this will be stored from the cache manager name = '_'.join([self._sensor, self._lpath, self._lrow, self._date]) untar_folder = config.io.cache.manager().register_item(name) # Check if we already unpacked this data all_files_present = False if os.path.exists(untar_folder): mtl_path = _find_mtl_file(untar_folder) if mtl_path: mtl_data = _parse_mtl_file(mtl_path) all_files_present = _check_if_files_present( mtl_data, untar_folder) if all_files_present: print('Already have unpacked files in ' + untar_folder) else: print('Unpacking tar file ' + paths + ' to folder ' + untar_folder) utilities.unpack_to_folder(paths, untar_folder) bands_to_use = _get_landsat_bands_to_use( self._sensor) if self._bands is None else self._bands # Generate all the band file names (the MTL file is not returned) self._mtl_path = _find_mtl_file(untar_folder) self._mtl_data = _parse_mtl_file(self._mtl_path) output_paths = _get_band_paths(self._mtl_data, untar_folder, bands_to_use) # Check that the files exist for p in output_paths: if not os.path.exists(p): raise Exception('Did not find expected file: ' + p + ' after unpacking tar file ' + paths) return output_paths
def _unpack(self, paths): # Get the folder where this will be stored from the cache manager name = '_'.join([self._sensor, self._date]) unpack_folder = config.io.cache.manager().register_item(name) # Check if we already unpacked this data (tif_path, imd_path) = _get_files_from_unpack_folder(unpack_folder) if imd_path and tif_path: #print('Already have unpacked files in ' + unpack_folder) pass else: print('Unpacking file ' + paths + ' to folder ' + unpack_folder) utilities.unpack_to_folder(paths, unpack_folder) (tif_path, imd_path) = _get_files_from_unpack_folder(unpack_folder) return (tif_path, imd_path)
def unpack_inputs(tar_folder, unpack_folder): """Make sure all of the input label files are untarred. The unpack folder can be the same as the tar folder. Returns the list of label files. """ if not os.path.exists(unpack_folder): os.mkdir(unpack_folder) file_list = [] # Loop through tar files input_list = os.listdir(tar_folder) for f in input_list: ext = os.path.splitext(f)[1] if ext != '.tar': continue # The name of the input tar does not fully match the untar file names name = os.path.basename(f) parts = name.split('_') prefix = '_'.join(parts[0:4]) # Look to see if we have a matching label file tar_path = os.path.join(tar_folder, f) label_path = look_for_file(unpack_folder, [prefix, '_INWM.tif']) # If we did not find the INWM file, untar. if not label_path: utilities.unpack_to_folder(tar_path, unpack_folder) # Look again for a matching INWM file label_path = look_for_file(unpack_folder, [prefix, '_INWM.tif']) if not label_path: raise Exception('Failed to untar label file: ' + tar_path) file_list.append(label_path) return file_list
def unpack_s1_to_folder(zip_path, unpack_folder): '''Returns the merged image path from the unpack folder. Unpacks the zip file and merges the source images as needed.''' if shutil.which('gpt') is None: raise Exception('Could not find the tool "gpt", do you have ESA SNAP installed and on your PATH?') with portalocker.Lock(zip_path, 'r', timeout=300) as unused: #pylint: disable=W0612 merged_path = get_merged_path(unpack_folder) try: test_image = tiff.TiffImage(merged_path) #pylint: disable=W0612 except Exception: #pylint: disable=W0703 test_image = None if test_image: # Merged image is ready to use print('Already have unpacked files in ' + unpack_folder) return merged_path # Otherwise go through the entire unpack process NUM_SOURCE_CHANNELS = 2 need_to_unpack = True if os.path.exists(unpack_folder): source_image_paths = get_files_from_unpack_folder(unpack_folder) if len(source_image_paths) == NUM_SOURCE_CHANNELS: need_to_unpack = False print('Already have files') else: print('Clearing unpack folder missing image files.') os.system('rm -rf ' + unpack_folder) if need_to_unpack: print('Unpacking file ' + zip_path + ' to folder ' + unpack_folder) utilities.unpack_to_folder(zip_path, unpack_folder) subdirs = os.listdir(unpack_folder) if len(subdirs) != 1: raise Exception('Unexpected Sentinel1 subdirectories: ' + str(subdirs)) cmd = 'mv ' + os.path.join(unpack_folder, subdirs[0]) +'/* ' + unpack_folder print(cmd) os.system(cmd) source_image_paths = get_files_from_unpack_folder(unpack_folder) if len(source_image_paths) != NUM_SOURCE_CHANNELS: raise Exception('Did not find two image files in ' + zip_path) USE_SNAP = True # To get real results we need to use SNAP if USE_SNAP: # Requires the Sentinel1 processing software to be installed # Run the preconfigured SNAP preprocessing graph # - The SNAP tool *must* write to a .tif extension, so we have to # rename the file if we want something else. temp_out_path = merged_path.replace('.vrt', '.tif') run_ffilipponi_preprocessing(unpack_folder, temp_out_path) dimap_path = temp_out_path + '.dim' cmd = 'pconvert -s 0,0 -f GeoTIFF-BigTiff -o ' + os.path.dirname(temp_out_path) +' '+ dimap_path print(cmd) os.system(cmd) MIN_IMAGE_SIZE = 1024*1024*500 # 500 MB, expected size is much larger if not os.path.exists(temp_out_path): raise Exception('Failed to run ESA SNAP preprocessing!') if os.path.getsize(temp_out_path) < MIN_IMAGE_SIZE: raise Exception('SNAP encountered a problem processing the file!') os.system('mv ' + temp_out_path + ' ' + merged_path) else: # Generate a merged file containing all input images as an N channel image cmd = 'gdalbuildvrt -separate ' + merged_path for f in source_image_paths: cmd += ' ' + f print(cmd) os.system(cmd) # Verify that we generated a valid image file try: test_image = tiff.TiffImage(merged_path) #pylint: disable=W0612 except Exception as e: #pylint: disable=W0703 raise Exception('Failed to generate merged Sentinel1 file: ' + merged_path) from e return merged_path