def _get_landsat_image(self):
l8 = Landsat8(self.landsat)
bounds = RasterBounds(
affine_transform=l8.rasterio_geometry['transform'],
profile=l8.rasterio_geometry)
proj_bounds = bounds.to_epsg(32611)
clip = proj_bounds.get_shapely_polygon()
return l8.rasterio_geometry, bounds, clip
def __init__(self,
target_profile=None,
year=None,
out_dir=None,
from_file=None):
self.url_base = 'https://nassgeodata.gmu.edu/axis2/services/CDLService/' \
'GetCDLFile?year={year}&bbox={wsen}'
if from_file:
self.from_file = from_file
with rasopen(from_file) as src:
self.cdl = src.read()
self.target_profile = src.profile
self.cdl_empty = False
else:
self.cdl_empty = True
self.cdl = None
if not out_dir:
self.cdl_location = os.path.join(os.path.dirname(__file__),
'model_data')
else:
self.cdl_location = out_dir
self.zip_file = os.path.join(self.cdl_location,
'{}_30m_cdls.zip'.format(year))
self.temp_dir = mkdtemp()
if target_profile and year:
self.target_profile = target_profile
self.bbox = RasterBounds(
profile=self.target_profile,
affine_transform=self.target_profile['transform'])
self.bbox.expand(**{
'east': 0.1,
'west': -0.1,
'north': 0.2,
'south': -0.2
})
self.bbox_projected = bb = self.bbox.to_lambert_conformal_conic(
)
bb_str = '{},{},{},{}'.format(bb[0], bb[1], bb[2], bb[3])
self.request_url = self.url_base.format(year=year, wsen=bb_str)
self.data_url = self._get_data_url()
self.original_tif = None
self.mask = None
self.projection = None
self.reprojection = None
def get_terrain(self):
slope_name = os.path.join(self.root, 'slope.tif')
aspect_name = os.path.join(self.root, 'aspect.tif')
dif_elev = os.path.join(self.root, 'elevation_diff.tif')
check = [
os.path.isfile(x) for x in [slope_name, aspect_name, dif_elev]
]
if False in check:
polygon = self.landsat.get_tile_geometry()
bb = RasterBounds(affine_transform=self.profile['transform'],
profile=self.profile,
latlon=True)
dem = AwsDem(zoom=10,
target_profile=self.profile,
bounds=bb,
clip_object=polygon)
dem.terrain(attribute='slope',
out_file=slope_name,
save_and_return=True)
dem.terrain(attribute='aspect',
out_file=aspect_name,
save_and_return=True)
elev = dem.terrain(attribute='elevation')
elev = elev - mean(elev)
dem.save(elev,
geometry=dem.target_profile,
output_filename=dif_elev)
def get_terrain(self):
"""
Get digital elevation maps from amazon web services
save in the project root directory with filenames enumerated
in the next three lines.
"""
slope_name = os.path.join(self.root, 'slope.tif')
aspect_name = os.path.join(self.root, 'aspect.tif')
dif_elev = os.path.join(self.root, 'elevation_diff.tif')
check = [os.path.isfile(x) for x in [slope_name, aspect_name, dif_elev]]
if False in check:
polygon = self.landsat.get_tile_geometry()
bb = RasterBounds(affine_transform=self.profile['transform'],
profile=self.profile, latlon=True)
dem = AwsDem(zoom=10, target_profile=self.profile, bounds=bb,
clip_object=polygon)
dem.terrain(attribute='slope',
out_file=slope_name, save_and_return=True)
dem.terrain(attribute='aspect',
out_file=aspect_name, save_and_return=True)
elev = dem.terrain(attribute='elevation')
elev = elev - mean(elev)
dem.save(elev, geometry=dem.target_profile, output_filename=dif_elev)
def test_conforming_array(self):
""" Test Thredds.TopoWx conforming array has equal shape to Landsat image
:return:
"""
home = os.path.expanduser('~')
tif_dir = os.path.join(home, 'images', 'LT5', 'image_test',
'full_image')
for t in ['tmin', 'tmax']:
out_file = os.path.join(
home, 'images', 'sandbox', 'thredds',
'{}_{}_{}_{}.tif'.format(t, self.date.year, self.date.month,
self.date.day))
l5 = Landsat5(tif_dir)
polygon = l5.get_tile_geometry()
bounds = RasterBounds(affine_transform=l5.transform,
profile=l5.profile,
latlon=True)
topowx = TopoWX(date=self.date,
bbox=bounds,
target_profile=l5.profile,
clip_feature=polygon)
temp = topowx.get_data_subset(grid_conform=True,
var=t,
out_file=out_file)
self.assertEqual(temp.shape, self.image_shape)
def test_elevation(self):
l8 = Landsat8(self.dir_name_LC8)
polygon = l8.get_tile_geometry()
bounds = RasterBounds(affine_transform=l8.rasterio_geometry['transform'], profile=l8.rasterio_geometry)
gridmet = GridMet('elev', date=self.date, bbox=bounds,
target_profile=l8.rasterio_geometry, clip_feature=polygon)
gridmet.save_raster()
gridmet.get_data_subset(os.path.join(self.grimet_raster_dir, 'elevation.tif'))
def test_other_raster_aspect(self):
with rasopen(self.raster) as src:
profile = src.profile
bound_box = RasterBounds(affine_transform=profile['affine'],
profile=profile)
dem = AwsDem(zoom=10, target_profile=profile, bounds=bound_box)
dem.terrain(attribute='aspect',
out_file='/home/dgketchum/IrrigationGIS/tests/mapzen_'
'image_clip_aspect.tif')
def test_conforming_array(self):
""" Test shape of Gridmet vs. Landsat image.
:return:
"""
l8 = Landsat8(self.dir_name_LC8)
polygon = l8.get_tile_geometry()
bounds = RasterBounds(affine_transform=l8.rasterio_geometry['transform'], profile=l8.rasterio_geometry)
gridmet = GridMet(self.var, date=self.date, bbox=bounds,
target_profile=l8.rasterio_geometry, clip_feature=polygon)
pet = gridmet.get_data_subset()
shape = 1, l8.rasterio_geometry['height'], l8.rasterio_geometry['width']
self.assertEqual(pet.shape, shape)
def __init__(self, image_id, image_dir, transform,
profile, clip_geo, date):
self.image_id = image_id
self.image_dir = image_dir
self.transform = transform
self.profile = profile
self.clip_geo = clip_geo
self.date = date
self.file_name = None
self.bounds = RasterBounds(affine_transform=self.transform,
profile=self.profile, latlon=True)
self.variable = None
self.file_path = None
self.shape = (1, profile['height'], profile['width'])
def test_conforming_array_local(self):
l8 = Landsat8(self.dir_name_LC8)
polygon = l8.get_tile_geometry()
bounds = RasterBounds(
affine_transform=l8.rasterio_geometry['transform'],
profile=l8.rasterio_geometry)
gridmet = GridMet(self.var,
date=self.date,
bbox=bounds,
target_profile=l8.rasterio_geometry,
clip_feature=polygon)
pet = gridmet.get_data_subset(
file_url='/home/dgketchum/Downloads/{}_{}.nc'.format(
self.var, self.date.year))
shape = 1, l8.rasterio_geometry['height'], l8.rasterio_geometry[
'width']
self.assertEqual(pet.shape, shape)
def test_conforming_array_to_native(self):
""" Test confoming array to native Gridmet raster.
Conforming array is what Thredds.Gridmet will build given geometry
paramters derived from LandsatImage object. This test builds that
array, and then compares it with several day's of native Gridmet netcdf
data. The rasters can't align perfectly, as the grid has been resampled.
This is built to look up 30 points, extract their location, get the native
raster value (i.e., geo) at that location, and the (local) conforming
array value. The ratio of the means from each raster must be w/in
0.5%.
:return:
"""
l8 = Landsat8(self.dir_name_LC8)
polygon = l8.get_tile_geometry()
bounds = RasterBounds(
affine_transform=l8.rasterio_geometry['transform'],
profile=l8.rasterio_geometry,
latlon=True)
for day in rrule(DAILY, dtstart=self.start, until=self.end):
gridmet = GridMet(self.var,
date=day,
bbox=bounds,
target_profile=l8.rasterio_geometry,
clip_feature=polygon)
date_str = datetime.strftime(day, '%Y-%m-%d')
met_arr = os.path.join(self.grimet_raster_dir,
'met_{}_{}.tif'.format(date_str, self.var))
met = gridmet.get_data_subset(out_filename=met_arr)
native = os.path.join(self.grimet_raster_dir,
'{}_pet.tif'.format(date_str))
points_dict = multi_raster_point_extract(
local_raster=met_arr,
geographic_raster=native,
points=self.scene_points,
image_profile=l8.rasterio_geometry)
geo_list, local_list = [], []
for key, val in points_dict.items():
geo_list.append(val['geo_val'])
local_list.append(val['local_val'])
ratio = mean(geo_list) / mean(local_list)
print('Ratio on {} of CONUSRaster:LocalRaster calculated is {}.'.
format(datetime.strftime(day, '%Y-%m-%d'), ratio))
self.assertAlmostEqual(ratio, 1.0, delta=0.005)
os.remove(met_arr)
def test_dem(self):
l8 = Landsat8(self.dir_name_LC8)
polygon = l8.get_tile_geometry()
profile = l8.rasterio_geometry
bb = RasterBounds(affine_transform=profile['affine'],
profile=profile,
latlon=True)
dem = AwsDem(zoom=10,
target_profile=profile,
bounds=bb,
clip_object=polygon)
elev = dem.terrain(
attribute='slope',
out_file='/home/dgketchum/IrrigationGIS/tests/mapzen_'
'{}_{}.tif'.format(l8.target_wrs_path, l8.target_wrs_row))
self.assertEqual(elev.shape, (1, 7429, 8163))
aspect = dem.terrain(attribute='aspect')
self.assertEqual(aspect.shape, (7429, 8163))
slope = dem.terrain(attribute='slope')
self.assertEqual(slope.shape, (1, 7429, 8163))
class CropDataLayer(object):
def __init__(self,
target_profile=None,
year=None,
out_dir=None,
from_file=None):
self.url_base = 'https://nassgeodata.gmu.edu/axis2/services/CDLService/' \
'GetCDLFile?year={year}&bbox={wsen}'
if from_file:
self.from_file = from_file
with rasopen(from_file) as src:
self.cdl = src.read()
self.target_profile = src.profile
self.cdl_empty = False
else:
self.cdl_empty = True
self.cdl = None
if not out_dir:
self.cdl_location = os.path.join(os.path.dirname(__file__),
'model_data')
else:
self.cdl_location = out_dir
self.zip_file = os.path.join(self.cdl_location,
'{}_30m_cdls.zip'.format(year))
self.temp_dir = mkdtemp()
if target_profile and year:
self.target_profile = target_profile
self.bbox = RasterBounds(
profile=self.target_profile,
affine_transform=self.target_profile['transform'])
self.bbox.expand(**{
'east': 0.1,
'west': -0.1,
'north': 0.2,
'south': -0.2
})
self.bbox_projected = bb = self.bbox.to_lambert_conformal_conic(
)
bb_str = '{},{},{},{}'.format(bb[0], bb[1], bb[2], bb[3])
self.request_url = self.url_base.format(year=year, wsen=bb_str)
self.data_url = self._get_data_url()
self.original_tif = None
self.mask = None
self.projection = None
self.reprojection = None
def get_original_tif(self, out_file=None):
req = get(self.data_url, verify=False)
if req.status_code != 200:
raise ValueError('Bad response {} from request.'.format(
req.status_code))
if not out_file:
self.original_tif = os.path.join(self.temp_dir,
os.path.basename(self.data_url))
else:
self.original_tif = out_file
with open(self.original_tif, 'wb') as f:
print('Downloading {}'.format(self.data_url))
for chunk in req.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
def get_conforming_data(self,
clip_geometry,
keep_original=False,
out_file=None):
self.get_original_tif()
self._reproject()
self._mask(clip_geometry)
result = self._resample()
if not keep_original:
os.remove(self.original_tif)
if out_file:
self.save(result,
self.target_profile,
output_filename=os.path.join(self.cdl_location,
'cdl.tif'))
self.cdl = result
return result
def get_mask(self, clip_geometry=None, out_file=None):
arr = None
if self.cdl_empty:
try:
arr = self.get_conforming_data(clip_geometry=clip_geometry)
except ValueError:
print('Need clip geometry to build cdl')
else:
arr = self.cdl
crop = list(self.crop.keys())
msk = isin(arr, crop)
msk = ~msk
msk = msk.astype(uint8)
profile = copy.deepcopy(self.target_profile)
profile['dtype'] = uint8
if out_file:
with rasopen(out_file, 'w', **profile) as dst:
dst.write(msk)
return msk
def _reproject(self):
self.reprojection = os.path.join(self.temp_dir, 'cdl_reprojection.tif')
with rasopen(self.original_tif, 'r') as src:
src_profile = src.profile
src_bounds = src.bounds
src_array = src.read(1)
dst_profile = copy.deepcopy(self.target_profile)
dst_profile['dtype'] = float32
bounds = src_bounds
dst_affine, dst_width, dst_height = cdt(src_profile['crs'],
dst_profile['crs'],
src_profile['width'],
src_profile['height'], *bounds)
dst_profile.update({
'crs': dst_profile['crs'],
'transform': dst_affine,
'width': dst_width,
'height': dst_height
})
with rasopen(self.reprojection, 'w', **dst_profile) as dst:
dst_array = empty((1, dst_height, dst_width), dtype=float32)
reproject(src_array,
dst_array,
src_transform=src_profile['transform'],
src_crs=src_profile['crs'],
dst_crs=self.target_profile['crs'],
dst_transform=dst_affine,
resampling=Resampling.nearest,
num_threads=2)
dst.write(
dst_array.reshape(1, dst_array.shape[1], dst_array.shape[2]))
def _mask(self, clip):
mask_path = os.path.join(self.temp_dir, 'masked.tif')
with rasopen(self.reprojection) as src:
out_arr, out_trans = mask(src, clip, crop=True, all_touched=True)
out_meta = src.meta.copy()
out_meta.update({
'driver': 'GTiff',
'height': out_arr.shape[1],
'width': out_arr.shape[2],
'transform': out_trans
})
with rasopen(mask_path, 'w', **out_meta) as dst:
dst.write(out_arr)
setattr(self, 'mask', mask_path)
delattr(self, 'reprojection')
def _resample(self):
resample_path = os.path.join(self.temp_dir, 'resample.tif')
with rasopen(self.mask, 'r') as src:
array = src.read(1)
profile = src.profile
res = src.res
try:
target_affine = self.target_profile['affine']
except KeyError:
target_affine = self.target_profile['transform']
target_res = target_affine.a
res_coeff = res[0] / target_res
new_array = empty(shape=(1, round(array.shape[0] * res_coeff),
round(array.shape[1] * res_coeff)),
dtype=float32)
aff = src.transform
new_affine = Affine(aff.a / res_coeff, aff.b, aff.c, aff.d,
aff.e / res_coeff, aff.f)
profile['transform'] = self.target_profile['transform']
profile['width'] = self.target_profile['width']
profile['height'] = self.target_profile['height']
profile['dtype'] = str(new_array.dtype)
delattr(self, 'mask')
with rasopen(resample_path, 'w', **profile) as dst:
reproject(array,
new_array,
src_transform=aff,
dst_transform=new_affine,
src_crs=src.crs,
dst_crs=src.crs,
resampling=Resampling.nearest)
dst.write(new_array)
with rasopen(resample_path, 'r') as src:
arr = src.read()
return arr
@staticmethod
def save(array, geometry, output_filename, crs=None, return_array=False):
try:
array = array.reshape(1, array.shape[1], array.shape[2])
except IndexError:
array = array.reshape(1, array.shape[0], array.shape[1])
geometry['dtype'] = str(array.dtype)
if crs:
geometry['crs'] = CRS({'init': crs})
with rasopen(output_filename, 'w', **geometry) as dst:
dst.write(array)
if return_array:
return array
return None
def download_zipped_cdl(self):
if not os.path.isfile(self.zip_file):
req = urlretrieve(self.request_url, self.cdl_location)
if req.status_code != 200:
raise ValueError('Bad response {} from request.'.format(
req.status_code))
with open(self.zip_file, 'wb') as f:
print('Downloading {}'.format(self.request_url))
for chunk in req.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
def _get_data_url(self):
r = get(self.request_url, verify=False)
tree = ElementTree.fromstring(r.content)
u = [ElementTree.tostring(e) for e in tree][0].decode("utf-8")
result = re.search('<returnURL>(.*)</returnURL>', u).group(1)
return result
@property
def crop(self):
return {
1: 'Corn',
2: 'Cotton',
3: 'Rice',
4: 'Sorghum',
5: 'Soybeans',
6: 'Sunflower',
10: 'Peanuts',
11: 'Tobacco',
12: 'Sweet Corn',
13: 'Pop or Orn Corn',
14: 'Mint',
21: 'Barley',
22: 'Durum Wheat',
23: 'Spring Wheat',
24: 'Winter Wheat',
25: 'Other Small Grains',
26: 'Dbl Crop WinWht / Soybeans',
27: 'Rye',
28: 'Oats',
29: 'Millet',
30: 'Speltz',
31: 'Canola',
32: 'Flaxseed',
33: 'Safflower',
34: 'Rape Seed',
35: 'Mustard',
36: 'Alfalfa',
37: 'Other Hay / NonAlfalfa',
38: 'Camelina',
39: 'Buckwheat',
41: 'Sugarbeets',
42: 'Dry Beans',
43: 'Potatoes',
44: 'Other Crops',
45: 'Sugarcane',
46: 'Sweet Potatoes',
47: 'Misc Vegs & Fruits',
48: 'Watermelons',
49: 'Onions',
50: 'Cucumbers',
51: 'Chick Peas',
52: 'Lentils',
53: 'Peas',
54: 'Tomatoes',
55: 'Caneberries',
56: 'Hops',
57: 'Herbs',
58: 'Clover/Wildflowers',
61: 'Fallow/Idle Cropland',
66: 'Cherries',
67: 'Peaches',
68: 'Apples',
69: 'Grapes',
70: 'Christmas Trees',
71: 'Other Tree Crops',
72: 'Citrus',
74: 'Pecans',
75: 'Almonds',
76: 'Walnuts',
77: 'Pears',
204: 'Pistachios',
205: 'Triticale',
206: 'Carrots',
207: 'Asparagus',
208: 'Garlic',
209: 'Cantaloupes',
210: 'Prunes',
211: 'Olives',
212: 'Oranges',
213: 'Honeydew Melons',
214: 'Broccoli',
216: 'Peppers',
217: 'Pomegranates',
218: 'Nectarines',
219: 'Greens',
220: 'Plums',
221: 'Strawberries',
222: 'Squash',
223: 'Apricots',
224: 'Vetch',
225: 'Dbl Crop WinWht/Corn',
226: 'Dbl Crop Oats/Corn',
227: 'Lettuce',
229: 'Pumpkins',
230: 'Dbl Crop Lettuce/Durum Wht',
231: 'Dbl Crop Lettuce/Cantaloupe',
232: 'Dbl Crop Lettuce/Cotton',
233: 'Dbl Crop Lettuce/Barley',
234: 'Dbl Crop Durum Wht/Sorghum',
235: 'Dbl Crop Barley/Sorghum',
236: 'Dbl Crop WinWht/Sorghum',
237: 'Dbl Crop Barley/Corn',
238: 'Dbl Crop WinWht/Cotton',
239: 'Dbl Crop Soybeans/Cotton',
240: 'Dbl Crop Soybeans/Oats',
241: 'Dbl Crop Corn/Soybeans',
242: 'Blueberries',
243: 'Cabbage',
244: 'Cauliflower',
245: 'Celery',
246: 'Radishes',
247: 'Turnips',
248: 'Eggplants',
249: 'Gourds',
250: 'Cranberries',
254: 'Dbl Crop Barley/Soybeans'
}
@property
def non_crop(self):
return {
37: 'Other Hay/Non Alfalfa',
59: 'Sod/Grass Seed',
60: 'Switchgrass',
63: 'Forest',
64: 'Shrubland',
65: 'Barren',
81: 'Clouds/No Data',
82: 'Developed',
83: 'Water',
87: 'Wetlands',
88: 'Nonag/Undefined',
92: 'Aquaculture',
111: 'Open Water',
112: 'Perennial Ice/Snow',
121: 'Developed/Open Space',
122: 'Developed/Low Intensity',
123: 'Developed/Med Intensity',
124: 'Developed/High Intensity',
131: 'Barren',
141: 'Deciduous Forest',
142: 'Evergreen Forest',
143: 'Mixed Forest',
152: 'Shrubland',
176: 'Grass/Pasture',
190: 'Woody Wetlands',
195: 'Herbaceous Wetlands'
}
def test_raster(self):
bb = RasterBounds(raster=self.image)
self.assertEqual(bb.get_nwse_tuple(), (47.61118761194992, -111.94851161596767,
47.54916612467683, -111.8361513308256))
def makePolyGeomFromRasterBounds(rastFn):
rast = rasterio.open(rastFn)
bb = rast.bounds
l = bb[0]
b = bb[1]
r = bb[2]
t = bb[3]
ul = (l,t)
ll = (l,b)
ur = (r,t)
lr = (r,l)
mpGeo = geometry.asPolygon((ul,ur,lr,ll))
bb = RasterBounds(affine_transform=profile['affine'],
profile=profile, latlon=True)
# open lake centriod shapefile
lakeCentShp = fiona.open(lakeCentShpFn)
nLake = len(lakeCentShp)
# get array of lake centroid coordinates
centArr = getShapefileCoordinatesFromMultipoint(lakeCentShpFn)
def __init__(self, obj):
'''
:param obj: Directory containing an unzipped Landsat 5, 7, or 8 image. This should include at least
a tif for each band, and a .mtl file.
'''
self.obj = obj
if os.path.isdir(obj):
self.isdir = True
self.date_acquired = None
self.file_list = os.listdir(obj)
self.tif_list = [x for x in os.listdir(obj) if x.endswith('.TIF')]
self.tif_list.sort()
# parse metadata file into attributes
# structure: {HEADER: {SUBHEADER: {key(attribute), val(attribute value)}}}
self.mtl = mtl.parsemeta(obj)
self.meta_header = list(self.mtl)[0]
self.super_dict = self.mtl[self.meta_header]
for key, val in self.super_dict.items():
for sub_key, sub_val in val.items():
# print(sub_key.lower(), sub_val)
setattr(self, sub_key.lower(), sub_val)
self.satellite = self.landsat_scene_id[:3]
# create numpy nd_array objects for each band
self.band_list = []
self.tif_dict = {}
for i, tif in enumerate(self.tif_list):
raster = os.path.join(self.obj, tif)
# set all lower case attributes
tif = tif.lower()
front_ind = tif.index('b')
end_ind = tif.index('.tif')
att_string = tif[front_ind:end_ind]
self.band_list.append(att_string)
self.tif_dict[att_string] = raster
self.band_count = i + 1
if i == 0:
with rasopen(raster) as src:
transform = src.transform
profile = src.profile
meta = src.meta.copy()
self.rasterio_geometry = meta
self.profile = profile
self.transform = transform
self.shape = (1, profile['height'], profile['width'])
bounds = RasterBounds(affine_transform=transform,
profile=profile,
latlon=False)
self.bounds = bounds
self.north, self.west, self.south, self.east = bounds.get_nwse_tuple(
)
self.coords = bounds.as_tuple('nsew')
self.solar_zenith = 90. - self.sun_elevation
self.solar_zenith_rad = self.solar_zenith * pi / 180
self.sun_elevation_rad = self.sun_elevation * pi / 180
self.earth_sun_dist = self.earth_sun_d(self.date_acquired)
dtime = datetime.strptime(str(self.date_acquired), '%Y-%m-%d')
julian_day = dtime.strftime('%j')
self.doy = int(julian_day)
self.scene_coords_deg = self._scene_centroid()
self.scene_coords_rad = deg2rad(self.scene_coords_deg[0]), deg2rad(
self.scene_coords_deg[1])
