def snow_and_ndsi_locations(src_info, no_data_value): """Generate Landsat snow locations and NDSI nodata locations Args: src_info <SourceInfo>: Information about the source data no_data_value <int>: No data (fill) value to use Returns: list(<int>): Locations where we decided snow exists list(<int>): Locations containing no data (fill) values """ logger = logging.getLogger(__name__) logger.info('Building TOA based NDSI band for Landsat data') # GREEN -------------------------------------------------------------- green_data = emis_util.extract_raster_data(src_info.toa.green.name, 1) green_no_data_locations = np.where(green_data == no_data_value) green_data = green_data * src_info.toa.green.scale_factor # SWIR1 -------------------------------------------------------------- swir1_data = emis_util.extract_raster_data(src_info.toa.swir1.name, 1) swir1_no_data_locations = np.where(swir1_data == no_data_value) swir1_data = swir1_data * src_info.toa.swir1.scale_factor # NDSI --------------------------------------------------------------- with np.errstate(divide='ignore'): ndsi_data = ((green_data - swir1_data) / (green_data + swir1_data)) # Cleanup no data locations ndsi_data[green_no_data_locations] = no_data_value ndsi_data[swir1_no_data_locations] = no_data_value # Memory cleanup del green_data del swir1_data del green_no_data_locations del swir1_no_data_locations # Capture ndsi no data locations ndsi_no_data_locations = np.where(ndsi_data == no_data_value) # Save the locations for the specfied snow pixels logger.info('Determine snow pixel locations') snow_locations = np.where(ndsi_data > 0.4) # Memory cleanup del ndsi_data return (snow_locations, ndsi_no_data_locations)
def extract_warped_data(ls_emis_stdev_warped_name, no_data_value, intermediate): """Retrieves the warped image data Args: ls_emis_stdev_warped_name <str>: Path to warped emissivity stdev file no_data_value <float>: Value to use for fill intermediate <bool>: Keep any intermediate products generated Returns: <numpy.2darray>: Emissivity standard deviation data list(<int>): Emissivity stdev locations containing no data (fill) values """ # Load the warped estimated Landsat EMIS stdev into memory ls_emis_stdev_data = emis_util.extract_raster_data( ls_emis_stdev_warped_name, 1) ls_emis_stdev_no_data_locations \ = np.where(ls_emis_stdev_data == no_data_value) if not intermediate: # Cleanup the intermediate files since we have them in memory if os.path.exists(ls_emis_stdev_warped_name): os.unlink(ls_emis_stdev_warped_name) return (ls_emis_stdev_data, ls_emis_stdev_no_data_locations)
def generate_landsat_ndvi(src_info, no_data_value): """Generate Landsat NDVI Args: src_info <SourceInfo>: Information about the source data no_data_value <int>: No data (fill) value to use Returns: <numpy.2darray>: Generated NDVI band data list(<int>): Locations containing no data (fill) values """ logger = logging.getLogger(__name__) logger.info('Building TOA based NDVI band for Landsat data') # NIR ---------------------------------------------------------------- nir_data = emis_util.extract_raster_data(src_info.toa.nir.name, 1) nir_no_data_locations = np.where(nir_data == no_data_value) nir_data = nir_data * src_info.toa.nir.scale_factor # RED ---------------------------------------------------------------- red_data = emis_util.extract_raster_data(src_info.toa.red.name, 1) red_no_data_locations = np.where(red_data == no_data_value) red_data = red_data * src_info.toa.red.scale_factor # NDVI --------------------------------------------------------------- ndvi_data = ((nir_data - red_data) / (nir_data + red_data)) # Cleanup no data locations ndvi_data[nir_no_data_locations] = no_data_value ndvi_data[red_no_data_locations] = no_data_value # Memory cleanup del red_data del nir_data del nir_no_data_locations del red_no_data_locations # Capture these before less than zero operation no_data_locations = np.where(ndvi_data == no_data_value) # Turn all negative values to zero # Use a realy small value so that we don't have negative zero (-0.0) ndvi_data[ndvi_data < 0.0000001] = 0 return (ndvi_data, no_data_locations)
def extract_warped_data(ls_emis_warped_name, aster_ndvi_warped_name, no_data_value, intermediate): """Retrieves the warped image data with some massaging of ASTER NDVI Args: ls_emis_warped_name <str>: Path to the warped emissivity file aster_ndvi_warped_name <str>: Path to the warped ASTER NDVI file no_data_value <float>: Value to use for fill intermediate <bool>: Keep any intermediate products generated Returns: <numpy.2darray>: Emissivity data list(<int>): Emissivity locations where gap data exists list(<int>): Emissivity locations containing no data (fill) values <numpy.2darray>: Emissivity standard deviation data <numpy.2darray>: ASTER NDVI data list(<int>): ASTER NDVI locations where gap data exists list(<int>): ASTER NDVI locations containing no data (fill) values """ # Load the warped estimated Landsat EMIS into memory ls_emis_data = emis_util.extract_raster_data(ls_emis_warped_name, 1) ls_emis_gap_locations = np.where(ls_emis_data == 0) ls_emis_no_data_locations = np.where(ls_emis_data == no_data_value) # Load the warped ASTER NDVI into memory aster_ndvi_data = emis_util.extract_raster_data(aster_ndvi_warped_name, 1) aster_ndvi_gap_locations = np.where(aster_ndvi_data == 0) aster_ndvi_no_data_locations = np.where(aster_ndvi_data == no_data_value) # Turn all negative values to zero # Use a realy small value so that we don't have negative zero (-0.0) aster_ndvi_data[aster_ndvi_data < 0.0000001] = 0 if not intermediate: # Cleanup the intermediate files since we have them in memory if os.path.exists(ls_emis_warped_name): os.unlink(ls_emis_warped_name) if os.path.exists(aster_ndvi_warped_name): os.unlink(aster_ndvi_warped_name) return (ls_emis_data, ls_emis_gap_locations, ls_emis_no_data_locations, aster_ndvi_data, aster_ndvi_gap_locations, aster_ndvi_no_data_locations)
def main(): setup_logging() object_store_url = str(get_env_var('OBJECT_STORE_URL', None)) sleep_range_min = int(get_env_var('SLEEP_RANGE_MIN', 1)) sleep_range_max = int(get_env_var('SLEEP_RANGE_MAX', 5)) data_range_min = int(get_env_var('DATA_RANGE_MIN', 0)) data_range_max = int(get_env_var('DATA_RANGE_MAX', 24872)) # We have 24873 items in our list so 0-24872 index range os.chdir('/mnt/mesos/sandbox') data = list() with open(FILENAME, 'r') as fd: data = [x.strip() for x in fd] while True: random.seed(time.gmtime()) index = random.randint(data_range_min, data_range_max) sleep_seconds = random.randint(sleep_range_min, sleep_range_max) # Add the bucket and filename to the url url_path = ''.join([object_store_url, '/AG100.003/', data[index]]) status_code = Web.http_transfer_file(url_path, data[index]) if status_code != requests.codes['ok']: if status_code != requests.codes['not_found']: raise Exception('HTTP - Transfer Failed') # Define the sub-dataset names emis_ds_name = ''.join(['HDF5:"', data[index], '"://Emissivity/Mean']) emis_sdev_ds_name = ''.join(['HDF5:"', data[index], '"://Emissivity/SDev']) ndvi_ds_name = ''.join(['HDF5:"', data[index], '"://NDVI/Mean']) lat_ds_name = ''.join(['HDF5:"', data[index], '"://Geolocation/Latitude']) lon_ds_name = ''.join(['HDF5:"', data[index], '"://Geolocation/Longitude']) aster_b13_data = emis_util.extract_raster_data(emis_ds_name, 4) aster_b14_data = emis_util.extract_raster_data(emis_ds_name, 5) aster_b13_sdev_data = emis_util.extract_raster_data(emis_sdev_ds_name, 4) aster_b14_sdev_data = emis_util.extract_raster_data(emis_sdev_ds_name, 5) aster_ndvi_data = emis_util.extract_raster_data(ndvi_ds_name, 1) aster_lat_data = emis_util.extract_raster_data(lat_ds_name, 1) aster_lon_data = emis_util.extract_raster_data(lon_ds_name, 1) time.sleep(sleep_seconds) os.unlink(data[index])
def extract_aster_data(url, filename): """Extracts the internal band(s) data for later processing Args: url <str>: URL to retrieve the file from filename <str>: Base HDF filename to extract from Returns: <numpy.2darray>: Mean Band 13 data <numpy.2darray>: Mean Band 14 data <numpy.2darray>: NDVI band data <int>: Samples in the data <int>: Lines in the data <2x3:float>: GDAL Affine transformation matrix [0] - Map X of upper left corner [1] - Pixel size in X direction [2] - Y rotation [3] - Map Y of upper left corner [4] - X rotation [5] - Pixel size in Y direction <bool>: True if the ASTER tile is available, False otherwise """ logger = logging.getLogger(__name__) # Build the HDF5 filename for the tile h5_file_path = ''.join([filename, '.h5']) emis_util.download_aster_ged_tile(url=url, h5_file_path=h5_file_path) # There are cases where the emissivity data will not be available # (for example, in water regions). aster_b13_data = [] aster_b14_data = [] aster_ndvi_data = [] samps = 0 lines = 0 geo_transform = [] if not os.path.exists(h5_file_path): # The ASTER tile is not available, so don't try to process it return (aster_b13_data, aster_b14_data, aster_ndvi_data, samps, lines, geo_transform, False) # Define the sub-dataset names emis_ds_name = ''.join(['HDF5:"', h5_file_path, '"://Emissivity/Mean']) ndvi_ds_name = ''.join(['HDF5:"', h5_file_path, '"://NDVI/Mean']) lat_ds_name = ''.join(['HDF5:"', h5_file_path, '"://Geolocation/Latitude']) lon_ds_name = ''.join(['HDF5:"', h5_file_path, '"://Geolocation/Longitude']) logger.debug(emis_ds_name) logger.debug(ndvi_ds_name) logger.debug(lat_ds_name) logger.debug(lon_ds_name) aster_b13_data = emis_util.extract_raster_data(emis_ds_name, 4) aster_b14_data = emis_util.extract_raster_data(emis_ds_name, 5) aster_ndvi_data = emis_util.extract_raster_data(ndvi_ds_name, 1) aster_lat_data = emis_util.extract_raster_data(lat_ds_name, 1) aster_lon_data = emis_util.extract_raster_data(lon_ds_name, 1) # Determine the minimum and maximum latitude and longitude x_min = aster_lon_data.min() x_max = aster_lon_data.max() y_min = aster_lat_data.min() y_max = aster_lat_data.max() del aster_lon_data del aster_lat_data # Determine the resolution and dimensions of the ASTER data (x_res, y_res, samps, lines) = ( emis_util.data_resolution_and_size(lat_ds_name, x_min, x_max, y_min, y_max)) # Build the geo transform geo_transform = [x_min, x_res, 0, y_max, 0, -y_res] return (aster_b13_data, aster_b14_data, aster_ndvi_data, samps, lines, geo_transform, True)