def snow_and_ndsi_locations(src_info, no_data_value):
"""Generate Landsat snow locations and NDSI nodata locations
Args:
src_info <SourceInfo>: Information about the source data
no_data_value <int>: No data (fill) value to use
Returns:
list(<int>): Locations where we decided snow exists
list(<int>): Locations containing no data (fill) values
"""
logger = logging.getLogger(__name__)
logger.info('Building TOA based NDSI band for Landsat data')
# GREEN --------------------------------------------------------------
green_data = emis_util.extract_raster_data(src_info.toa.green.name, 1)
green_no_data_locations = np.where(green_data == no_data_value)
green_data = green_data * src_info.toa.green.scale_factor
# SWIR1 --------------------------------------------------------------
swir1_data = emis_util.extract_raster_data(src_info.toa.swir1.name, 1)
swir1_no_data_locations = np.where(swir1_data == no_data_value)
swir1_data = swir1_data * src_info.toa.swir1.scale_factor
# NDSI ---------------------------------------------------------------
with np.errstate(divide='ignore'):
ndsi_data = ((green_data - swir1_data) / (green_data + swir1_data))
# Cleanup no data locations
ndsi_data[green_no_data_locations] = no_data_value
ndsi_data[swir1_no_data_locations] = no_data_value
# Memory cleanup
del green_data
del swir1_data
del green_no_data_locations
del swir1_no_data_locations
# Capture ndsi no data locations
ndsi_no_data_locations = np.where(ndsi_data == no_data_value)
# Save the locations for the specfied snow pixels
logger.info('Determine snow pixel locations')
snow_locations = np.where(ndsi_data > 0.4)
# Memory cleanup
del ndsi_data
return (snow_locations, ndsi_no_data_locations)
def extract_warped_data(ls_emis_stdev_warped_name, no_data_value, intermediate):
"""Retrieves the warped image data
Args:
ls_emis_stdev_warped_name <str>: Path to warped emissivity stdev file
no_data_value <float>: Value to use for fill
intermediate <bool>: Keep any intermediate products generated
Returns:
<numpy.2darray>: Emissivity standard deviation data
list(<int>): Emissivity stdev locations containing no data (fill) values
"""
# Load the warped estimated Landsat EMIS stdev into memory
ls_emis_stdev_data = emis_util.extract_raster_data(
ls_emis_stdev_warped_name, 1)
ls_emis_stdev_no_data_locations \
= np.where(ls_emis_stdev_data == no_data_value)
if not intermediate:
# Cleanup the intermediate files since we have them in memory
if os.path.exists(ls_emis_stdev_warped_name):
os.unlink(ls_emis_stdev_warped_name)
return (ls_emis_stdev_data, ls_emis_stdev_no_data_locations)
def generate_landsat_ndvi(src_info, no_data_value):
"""Generate Landsat NDVI
Args:
src_info <SourceInfo>: Information about the source data
no_data_value <int>: No data (fill) value to use
Returns:
<numpy.2darray>: Generated NDVI band data
list(<int>): Locations containing no data (fill) values
"""
logger = logging.getLogger(__name__)
logger.info('Building TOA based NDVI band for Landsat data')
# NIR ----------------------------------------------------------------
nir_data = emis_util.extract_raster_data(src_info.toa.nir.name, 1)
nir_no_data_locations = np.where(nir_data == no_data_value)
nir_data = nir_data * src_info.toa.nir.scale_factor
# RED ----------------------------------------------------------------
red_data = emis_util.extract_raster_data(src_info.toa.red.name, 1)
red_no_data_locations = np.where(red_data == no_data_value)
red_data = red_data * src_info.toa.red.scale_factor
# NDVI ---------------------------------------------------------------
ndvi_data = ((nir_data - red_data) / (nir_data + red_data))
# Cleanup no data locations
ndvi_data[nir_no_data_locations] = no_data_value
ndvi_data[red_no_data_locations] = no_data_value
# Memory cleanup
del red_data
del nir_data
del nir_no_data_locations
del red_no_data_locations
# Capture these before less than zero operation
no_data_locations = np.where(ndvi_data == no_data_value)
# Turn all negative values to zero
# Use a realy small value so that we don't have negative zero (-0.0)
ndvi_data[ndvi_data < 0.0000001] = 0
return (ndvi_data, no_data_locations)
def extract_warped_data(ls_emis_warped_name, aster_ndvi_warped_name,
no_data_value, intermediate):
"""Retrieves the warped image data with some massaging of ASTER NDVI
Args:
ls_emis_warped_name <str>: Path to the warped emissivity file
aster_ndvi_warped_name <str>: Path to the warped ASTER NDVI file
no_data_value <float>: Value to use for fill
intermediate <bool>: Keep any intermediate products generated
Returns:
<numpy.2darray>: Emissivity data
list(<int>): Emissivity locations where gap data exists
list(<int>): Emissivity locations containing no data (fill) values
<numpy.2darray>: Emissivity standard deviation data
<numpy.2darray>: ASTER NDVI data
list(<int>): ASTER NDVI locations where gap data exists
list(<int>): ASTER NDVI locations containing no data (fill) values
"""
# Load the warped estimated Landsat EMIS into memory
ls_emis_data = emis_util.extract_raster_data(ls_emis_warped_name, 1)
ls_emis_gap_locations = np.where(ls_emis_data == 0)
ls_emis_no_data_locations = np.where(ls_emis_data == no_data_value)
# Load the warped ASTER NDVI into memory
aster_ndvi_data = emis_util.extract_raster_data(aster_ndvi_warped_name, 1)
aster_ndvi_gap_locations = np.where(aster_ndvi_data == 0)
aster_ndvi_no_data_locations = np.where(aster_ndvi_data == no_data_value)
# Turn all negative values to zero
# Use a realy small value so that we don't have negative zero (-0.0)
aster_ndvi_data[aster_ndvi_data < 0.0000001] = 0
if not intermediate:
# Cleanup the intermediate files since we have them in memory
if os.path.exists(ls_emis_warped_name):
os.unlink(ls_emis_warped_name)
if os.path.exists(aster_ndvi_warped_name):
os.unlink(aster_ndvi_warped_name)
return (ls_emis_data, ls_emis_gap_locations, ls_emis_no_data_locations,
aster_ndvi_data, aster_ndvi_gap_locations,
aster_ndvi_no_data_locations)
def main():
setup_logging()
object_store_url = str(get_env_var('OBJECT_STORE_URL', None))
sleep_range_min = int(get_env_var('SLEEP_RANGE_MIN', 1))
sleep_range_max = int(get_env_var('SLEEP_RANGE_MAX', 5))
data_range_min = int(get_env_var('DATA_RANGE_MIN', 0))
data_range_max = int(get_env_var('DATA_RANGE_MAX', 24872))
# We have 24873 items in our list so 0-24872 index range
os.chdir('/mnt/mesos/sandbox')
data = list()
with open(FILENAME, 'r') as fd:
data = [x.strip() for x in fd]
while True:
random.seed(time.gmtime())
index = random.randint(data_range_min, data_range_max)
sleep_seconds = random.randint(sleep_range_min, sleep_range_max)
# Add the bucket and filename to the url
url_path = ''.join([object_store_url, '/AG100.003/', data[index]])
status_code = Web.http_transfer_file(url_path, data[index])
if status_code != requests.codes['ok']:
if status_code != requests.codes['not_found']:
raise Exception('HTTP - Transfer Failed')
# Define the sub-dataset names
emis_ds_name = ''.join(['HDF5:"', data[index], '"://Emissivity/Mean'])
emis_sdev_ds_name = ''.join(['HDF5:"', data[index], '"://Emissivity/SDev'])
ndvi_ds_name = ''.join(['HDF5:"', data[index], '"://NDVI/Mean'])
lat_ds_name = ''.join(['HDF5:"', data[index], '"://Geolocation/Latitude'])
lon_ds_name = ''.join(['HDF5:"', data[index], '"://Geolocation/Longitude'])
aster_b13_data = emis_util.extract_raster_data(emis_ds_name, 4)
aster_b14_data = emis_util.extract_raster_data(emis_ds_name, 5)
aster_b13_sdev_data = emis_util.extract_raster_data(emis_sdev_ds_name, 4)
aster_b14_sdev_data = emis_util.extract_raster_data(emis_sdev_ds_name, 5)
aster_ndvi_data = emis_util.extract_raster_data(ndvi_ds_name, 1)
aster_lat_data = emis_util.extract_raster_data(lat_ds_name, 1)
aster_lon_data = emis_util.extract_raster_data(lon_ds_name, 1)
time.sleep(sleep_seconds)
os.unlink(data[index])
def extract_aster_data(url, filename):
"""Extracts the internal band(s) data for later processing
Args:
url <str>: URL to retrieve the file from
filename <str>: Base HDF filename to extract from
Returns:
<numpy.2darray>: Mean Band 13 data
<numpy.2darray>: Mean Band 14 data
<numpy.2darray>: NDVI band data
<int>: Samples in the data
<int>: Lines in the data
<2x3:float>: GDAL Affine transformation matrix
[0] - Map X of upper left corner
[1] - Pixel size in X direction
[2] - Y rotation
[3] - Map Y of upper left corner
[4] - X rotation
[5] - Pixel size in Y direction
<bool>: True if the ASTER tile is available, False otherwise
"""
logger = logging.getLogger(__name__)
# Build the HDF5 filename for the tile
h5_file_path = ''.join([filename, '.h5'])
emis_util.download_aster_ged_tile(url=url, h5_file_path=h5_file_path)
# There are cases where the emissivity data will not be available
# (for example, in water regions).
aster_b13_data = []
aster_b14_data = []
aster_ndvi_data = []
samps = 0
lines = 0
geo_transform = []
if not os.path.exists(h5_file_path):
# The ASTER tile is not available, so don't try to process it
return (aster_b13_data, aster_b14_data, aster_ndvi_data, samps, lines,
geo_transform, False)
# Define the sub-dataset names
emis_ds_name = ''.join(['HDF5:"', h5_file_path,
'"://Emissivity/Mean'])
ndvi_ds_name = ''.join(['HDF5:"', h5_file_path,
'"://NDVI/Mean'])
lat_ds_name = ''.join(['HDF5:"', h5_file_path,
'"://Geolocation/Latitude'])
lon_ds_name = ''.join(['HDF5:"', h5_file_path,
'"://Geolocation/Longitude'])
logger.debug(emis_ds_name)
logger.debug(ndvi_ds_name)
logger.debug(lat_ds_name)
logger.debug(lon_ds_name)
aster_b13_data = emis_util.extract_raster_data(emis_ds_name, 4)
aster_b14_data = emis_util.extract_raster_data(emis_ds_name, 5)
aster_ndvi_data = emis_util.extract_raster_data(ndvi_ds_name, 1)
aster_lat_data = emis_util.extract_raster_data(lat_ds_name, 1)
aster_lon_data = emis_util.extract_raster_data(lon_ds_name, 1)
# Determine the minimum and maximum latitude and longitude
x_min = aster_lon_data.min()
x_max = aster_lon_data.max()
y_min = aster_lat_data.min()
y_max = aster_lat_data.max()
del aster_lon_data
del aster_lat_data
# Determine the resolution and dimensions of the ASTER data
(x_res, y_res, samps, lines) = (
emis_util.data_resolution_and_size(lat_ds_name,
x_min, x_max, y_min, y_max))
# Build the geo transform
geo_transform = [x_min, x_res, 0, y_max, 0, -y_res]
return (aster_b13_data, aster_b14_data, aster_ndvi_data, samps, lines,
geo_transform, True)