def _get_landsat_image(self): l8 = Landsat8(self.landsat) bounds = RasterBounds( affine_transform=l8.rasterio_geometry['transform'], profile=l8.rasterio_geometry) proj_bounds = bounds.to_epsg(32611) clip = proj_bounds.get_shapely_polygon() return l8.rasterio_geometry, bounds, clip
def __init__(self, target_profile=None, year=None, out_dir=None, from_file=None): self.url_base = 'https://nassgeodata.gmu.edu/axis2/services/CDLService/' \ 'GetCDLFile?year={year}&bbox={wsen}' if from_file: self.from_file = from_file with rasopen(from_file) as src: self.cdl = src.read() self.target_profile = src.profile self.cdl_empty = False else: self.cdl_empty = True self.cdl = None if not out_dir: self.cdl_location = os.path.join(os.path.dirname(__file__), 'model_data') else: self.cdl_location = out_dir self.zip_file = os.path.join(self.cdl_location, '{}_30m_cdls.zip'.format(year)) self.temp_dir = mkdtemp() if target_profile and year: self.target_profile = target_profile self.bbox = RasterBounds( profile=self.target_profile, affine_transform=self.target_profile['transform']) self.bbox.expand(**{ 'east': 0.1, 'west': -0.1, 'north': 0.2, 'south': -0.2 }) self.bbox_projected = bb = self.bbox.to_lambert_conformal_conic( ) bb_str = '{},{},{},{}'.format(bb[0], bb[1], bb[2], bb[3]) self.request_url = self.url_base.format(year=year, wsen=bb_str) self.data_url = self._get_data_url() self.original_tif = None self.mask = None self.projection = None self.reprojection = None
def get_terrain(self): slope_name = os.path.join(self.root, 'slope.tif') aspect_name = os.path.join(self.root, 'aspect.tif') dif_elev = os.path.join(self.root, 'elevation_diff.tif') check = [ os.path.isfile(x) for x in [slope_name, aspect_name, dif_elev] ] if False in check: polygon = self.landsat.get_tile_geometry() bb = RasterBounds(affine_transform=self.profile['transform'], profile=self.profile, latlon=True) dem = AwsDem(zoom=10, target_profile=self.profile, bounds=bb, clip_object=polygon) dem.terrain(attribute='slope', out_file=slope_name, save_and_return=True) dem.terrain(attribute='aspect', out_file=aspect_name, save_and_return=True) elev = dem.terrain(attribute='elevation') elev = elev - mean(elev) dem.save(elev, geometry=dem.target_profile, output_filename=dif_elev)
def get_terrain(self): """ Get digital elevation maps from amazon web services save in the project root directory with filenames enumerated in the next three lines. """ slope_name = os.path.join(self.root, 'slope.tif') aspect_name = os.path.join(self.root, 'aspect.tif') dif_elev = os.path.join(self.root, 'elevation_diff.tif') check = [os.path.isfile(x) for x in [slope_name, aspect_name, dif_elev]] if False in check: polygon = self.landsat.get_tile_geometry() bb = RasterBounds(affine_transform=self.profile['transform'], profile=self.profile, latlon=True) dem = AwsDem(zoom=10, target_profile=self.profile, bounds=bb, clip_object=polygon) dem.terrain(attribute='slope', out_file=slope_name, save_and_return=True) dem.terrain(attribute='aspect', out_file=aspect_name, save_and_return=True) elev = dem.terrain(attribute='elevation') elev = elev - mean(elev) dem.save(elev, geometry=dem.target_profile, output_filename=dif_elev)
def test_conforming_array(self): """ Test Thredds.TopoWx conforming array has equal shape to Landsat image :return: """ home = os.path.expanduser('~') tif_dir = os.path.join(home, 'images', 'LT5', 'image_test', 'full_image') for t in ['tmin', 'tmax']: out_file = os.path.join( home, 'images', 'sandbox', 'thredds', '{}_{}_{}_{}.tif'.format(t, self.date.year, self.date.month, self.date.day)) l5 = Landsat5(tif_dir) polygon = l5.get_tile_geometry() bounds = RasterBounds(affine_transform=l5.transform, profile=l5.profile, latlon=True) topowx = TopoWX(date=self.date, bbox=bounds, target_profile=l5.profile, clip_feature=polygon) temp = topowx.get_data_subset(grid_conform=True, var=t, out_file=out_file) self.assertEqual(temp.shape, self.image_shape)
def test_elevation(self): l8 = Landsat8(self.dir_name_LC8) polygon = l8.get_tile_geometry() bounds = RasterBounds(affine_transform=l8.rasterio_geometry['transform'], profile=l8.rasterio_geometry) gridmet = GridMet('elev', date=self.date, bbox=bounds, target_profile=l8.rasterio_geometry, clip_feature=polygon) gridmet.save_raster() gridmet.get_data_subset(os.path.join(self.grimet_raster_dir, 'elevation.tif'))
def test_other_raster_aspect(self): with rasopen(self.raster) as src: profile = src.profile bound_box = RasterBounds(affine_transform=profile['affine'], profile=profile) dem = AwsDem(zoom=10, target_profile=profile, bounds=bound_box) dem.terrain(attribute='aspect', out_file='/home/dgketchum/IrrigationGIS/tests/mapzen_' 'image_clip_aspect.tif')
def test_conforming_array(self): """ Test shape of Gridmet vs. Landsat image. :return: """ l8 = Landsat8(self.dir_name_LC8) polygon = l8.get_tile_geometry() bounds = RasterBounds(affine_transform=l8.rasterio_geometry['transform'], profile=l8.rasterio_geometry) gridmet = GridMet(self.var, date=self.date, bbox=bounds, target_profile=l8.rasterio_geometry, clip_feature=polygon) pet = gridmet.get_data_subset() shape = 1, l8.rasterio_geometry['height'], l8.rasterio_geometry['width'] self.assertEqual(pet.shape, shape)
def __init__(self, image_id, image_dir, transform, profile, clip_geo, date): self.image_id = image_id self.image_dir = image_dir self.transform = transform self.profile = profile self.clip_geo = clip_geo self.date = date self.file_name = None self.bounds = RasterBounds(affine_transform=self.transform, profile=self.profile, latlon=True) self.variable = None self.file_path = None self.shape = (1, profile['height'], profile['width'])
def test_conforming_array_local(self): l8 = Landsat8(self.dir_name_LC8) polygon = l8.get_tile_geometry() bounds = RasterBounds( affine_transform=l8.rasterio_geometry['transform'], profile=l8.rasterio_geometry) gridmet = GridMet(self.var, date=self.date, bbox=bounds, target_profile=l8.rasterio_geometry, clip_feature=polygon) pet = gridmet.get_data_subset( file_url='/home/dgketchum/Downloads/{}_{}.nc'.format( self.var, self.date.year)) shape = 1, l8.rasterio_geometry['height'], l8.rasterio_geometry[ 'width'] self.assertEqual(pet.shape, shape)
def test_conforming_array_to_native(self): """ Test confoming array to native Gridmet raster. Conforming array is what Thredds.Gridmet will build given geometry paramters derived from LandsatImage object. This test builds that array, and then compares it with several day's of native Gridmet netcdf data. The rasters can't align perfectly, as the grid has been resampled. This is built to look up 30 points, extract their location, get the native raster value (i.e., geo) at that location, and the (local) conforming array value. The ratio of the means from each raster must be w/in 0.5%. :return: """ l8 = Landsat8(self.dir_name_LC8) polygon = l8.get_tile_geometry() bounds = RasterBounds( affine_transform=l8.rasterio_geometry['transform'], profile=l8.rasterio_geometry, latlon=True) for day in rrule(DAILY, dtstart=self.start, until=self.end): gridmet = GridMet(self.var, date=day, bbox=bounds, target_profile=l8.rasterio_geometry, clip_feature=polygon) date_str = datetime.strftime(day, '%Y-%m-%d') met_arr = os.path.join(self.grimet_raster_dir, 'met_{}_{}.tif'.format(date_str, self.var)) met = gridmet.get_data_subset(out_filename=met_arr) native = os.path.join(self.grimet_raster_dir, '{}_pet.tif'.format(date_str)) points_dict = multi_raster_point_extract( local_raster=met_arr, geographic_raster=native, points=self.scene_points, image_profile=l8.rasterio_geometry) geo_list, local_list = [], [] for key, val in points_dict.items(): geo_list.append(val['geo_val']) local_list.append(val['local_val']) ratio = mean(geo_list) / mean(local_list) print('Ratio on {} of CONUSRaster:LocalRaster calculated is {}.'. format(datetime.strftime(day, '%Y-%m-%d'), ratio)) self.assertAlmostEqual(ratio, 1.0, delta=0.005) os.remove(met_arr)
def test_dem(self): l8 = Landsat8(self.dir_name_LC8) polygon = l8.get_tile_geometry() profile = l8.rasterio_geometry bb = RasterBounds(affine_transform=profile['affine'], profile=profile, latlon=True) dem = AwsDem(zoom=10, target_profile=profile, bounds=bb, clip_object=polygon) elev = dem.terrain( attribute='slope', out_file='/home/dgketchum/IrrigationGIS/tests/mapzen_' '{}_{}.tif'.format(l8.target_wrs_path, l8.target_wrs_row)) self.assertEqual(elev.shape, (1, 7429, 8163)) aspect = dem.terrain(attribute='aspect') self.assertEqual(aspect.shape, (7429, 8163)) slope = dem.terrain(attribute='slope') self.assertEqual(slope.shape, (1, 7429, 8163))
class CropDataLayer(object): def __init__(self, target_profile=None, year=None, out_dir=None, from_file=None): self.url_base = 'https://nassgeodata.gmu.edu/axis2/services/CDLService/' \ 'GetCDLFile?year={year}&bbox={wsen}' if from_file: self.from_file = from_file with rasopen(from_file) as src: self.cdl = src.read() self.target_profile = src.profile self.cdl_empty = False else: self.cdl_empty = True self.cdl = None if not out_dir: self.cdl_location = os.path.join(os.path.dirname(__file__), 'model_data') else: self.cdl_location = out_dir self.zip_file = os.path.join(self.cdl_location, '{}_30m_cdls.zip'.format(year)) self.temp_dir = mkdtemp() if target_profile and year: self.target_profile = target_profile self.bbox = RasterBounds( profile=self.target_profile, affine_transform=self.target_profile['transform']) self.bbox.expand(**{ 'east': 0.1, 'west': -0.1, 'north': 0.2, 'south': -0.2 }) self.bbox_projected = bb = self.bbox.to_lambert_conformal_conic( ) bb_str = '{},{},{},{}'.format(bb[0], bb[1], bb[2], bb[3]) self.request_url = self.url_base.format(year=year, wsen=bb_str) self.data_url = self._get_data_url() self.original_tif = None self.mask = None self.projection = None self.reprojection = None def get_original_tif(self, out_file=None): req = get(self.data_url, verify=False) if req.status_code != 200: raise ValueError('Bad response {} from request.'.format( req.status_code)) if not out_file: self.original_tif = os.path.join(self.temp_dir, os.path.basename(self.data_url)) else: self.original_tif = out_file with open(self.original_tif, 'wb') as f: print('Downloading {}'.format(self.data_url)) for chunk in req.iter_content(chunk_size=1024): if chunk: f.write(chunk) def get_conforming_data(self, clip_geometry, keep_original=False, out_file=None): self.get_original_tif() self._reproject() self._mask(clip_geometry) result = self._resample() if not keep_original: os.remove(self.original_tif) if out_file: self.save(result, self.target_profile, output_filename=os.path.join(self.cdl_location, 'cdl.tif')) self.cdl = result return result def get_mask(self, clip_geometry=None, out_file=None): arr = None if self.cdl_empty: try: arr = self.get_conforming_data(clip_geometry=clip_geometry) except ValueError: print('Need clip geometry to build cdl') else: arr = self.cdl crop = list(self.crop.keys()) msk = isin(arr, crop) msk = ~msk msk = msk.astype(uint8) profile = copy.deepcopy(self.target_profile) profile['dtype'] = uint8 if out_file: with rasopen(out_file, 'w', **profile) as dst: dst.write(msk) return msk def _reproject(self): self.reprojection = os.path.join(self.temp_dir, 'cdl_reprojection.tif') with rasopen(self.original_tif, 'r') as src: src_profile = src.profile src_bounds = src.bounds src_array = src.read(1) dst_profile = copy.deepcopy(self.target_profile) dst_profile['dtype'] = float32 bounds = src_bounds dst_affine, dst_width, dst_height = cdt(src_profile['crs'], dst_profile['crs'], src_profile['width'], src_profile['height'], *bounds) dst_profile.update({ 'crs': dst_profile['crs'], 'transform': dst_affine, 'width': dst_width, 'height': dst_height }) with rasopen(self.reprojection, 'w', **dst_profile) as dst: dst_array = empty((1, dst_height, dst_width), dtype=float32) reproject(src_array, dst_array, src_transform=src_profile['transform'], src_crs=src_profile['crs'], dst_crs=self.target_profile['crs'], dst_transform=dst_affine, resampling=Resampling.nearest, num_threads=2) dst.write( dst_array.reshape(1, dst_array.shape[1], dst_array.shape[2])) def _mask(self, clip): mask_path = os.path.join(self.temp_dir, 'masked.tif') with rasopen(self.reprojection) as src: out_arr, out_trans = mask(src, clip, crop=True, all_touched=True) out_meta = src.meta.copy() out_meta.update({ 'driver': 'GTiff', 'height': out_arr.shape[1], 'width': out_arr.shape[2], 'transform': out_trans }) with rasopen(mask_path, 'w', **out_meta) as dst: dst.write(out_arr) setattr(self, 'mask', mask_path) delattr(self, 'reprojection') def _resample(self): resample_path = os.path.join(self.temp_dir, 'resample.tif') with rasopen(self.mask, 'r') as src: array = src.read(1) profile = src.profile res = src.res try: target_affine = self.target_profile['affine'] except KeyError: target_affine = self.target_profile['transform'] target_res = target_affine.a res_coeff = res[0] / target_res new_array = empty(shape=(1, round(array.shape[0] * res_coeff), round(array.shape[1] * res_coeff)), dtype=float32) aff = src.transform new_affine = Affine(aff.a / res_coeff, aff.b, aff.c, aff.d, aff.e / res_coeff, aff.f) profile['transform'] = self.target_profile['transform'] profile['width'] = self.target_profile['width'] profile['height'] = self.target_profile['height'] profile['dtype'] = str(new_array.dtype) delattr(self, 'mask') with rasopen(resample_path, 'w', **profile) as dst: reproject(array, new_array, src_transform=aff, dst_transform=new_affine, src_crs=src.crs, dst_crs=src.crs, resampling=Resampling.nearest) dst.write(new_array) with rasopen(resample_path, 'r') as src: arr = src.read() return arr @staticmethod def save(array, geometry, output_filename, crs=None, return_array=False): try: array = array.reshape(1, array.shape[1], array.shape[2]) except IndexError: array = array.reshape(1, array.shape[0], array.shape[1]) geometry['dtype'] = str(array.dtype) if crs: geometry['crs'] = CRS({'init': crs}) with rasopen(output_filename, 'w', **geometry) as dst: dst.write(array) if return_array: return array return None def download_zipped_cdl(self): if not os.path.isfile(self.zip_file): req = urlretrieve(self.request_url, self.cdl_location) if req.status_code != 200: raise ValueError('Bad response {} from request.'.format( req.status_code)) with open(self.zip_file, 'wb') as f: print('Downloading {}'.format(self.request_url)) for chunk in req.iter_content(chunk_size=1024): if chunk: f.write(chunk) def _get_data_url(self): r = get(self.request_url, verify=False) tree = ElementTree.fromstring(r.content) u = [ElementTree.tostring(e) for e in tree][0].decode("utf-8") result = re.search('<returnURL>(.*)</returnURL>', u).group(1) return result @property def crop(self): return { 1: 'Corn', 2: 'Cotton', 3: 'Rice', 4: 'Sorghum', 5: 'Soybeans', 6: 'Sunflower', 10: 'Peanuts', 11: 'Tobacco', 12: 'Sweet Corn', 13: 'Pop or Orn Corn', 14: 'Mint', 21: 'Barley', 22: 'Durum Wheat', 23: 'Spring Wheat', 24: 'Winter Wheat', 25: 'Other Small Grains', 26: 'Dbl Crop WinWht / Soybeans', 27: 'Rye', 28: 'Oats', 29: 'Millet', 30: 'Speltz', 31: 'Canola', 32: 'Flaxseed', 33: 'Safflower', 34: 'Rape Seed', 35: 'Mustard', 36: 'Alfalfa', 37: 'Other Hay / NonAlfalfa', 38: 'Camelina', 39: 'Buckwheat', 41: 'Sugarbeets', 42: 'Dry Beans', 43: 'Potatoes', 44: 'Other Crops', 45: 'Sugarcane', 46: 'Sweet Potatoes', 47: 'Misc Vegs & Fruits', 48: 'Watermelons', 49: 'Onions', 50: 'Cucumbers', 51: 'Chick Peas', 52: 'Lentils', 53: 'Peas', 54: 'Tomatoes', 55: 'Caneberries', 56: 'Hops', 57: 'Herbs', 58: 'Clover/Wildflowers', 61: 'Fallow/Idle Cropland', 66: 'Cherries', 67: 'Peaches', 68: 'Apples', 69: 'Grapes', 70: 'Christmas Trees', 71: 'Other Tree Crops', 72: 'Citrus', 74: 'Pecans', 75: 'Almonds', 76: 'Walnuts', 77: 'Pears', 204: 'Pistachios', 205: 'Triticale', 206: 'Carrots', 207: 'Asparagus', 208: 'Garlic', 209: 'Cantaloupes', 210: 'Prunes', 211: 'Olives', 212: 'Oranges', 213: 'Honeydew Melons', 214: 'Broccoli', 216: 'Peppers', 217: 'Pomegranates', 218: 'Nectarines', 219: 'Greens', 220: 'Plums', 221: 'Strawberries', 222: 'Squash', 223: 'Apricots', 224: 'Vetch', 225: 'Dbl Crop WinWht/Corn', 226: 'Dbl Crop Oats/Corn', 227: 'Lettuce', 229: 'Pumpkins', 230: 'Dbl Crop Lettuce/Durum Wht', 231: 'Dbl Crop Lettuce/Cantaloupe', 232: 'Dbl Crop Lettuce/Cotton', 233: 'Dbl Crop Lettuce/Barley', 234: 'Dbl Crop Durum Wht/Sorghum', 235: 'Dbl Crop Barley/Sorghum', 236: 'Dbl Crop WinWht/Sorghum', 237: 'Dbl Crop Barley/Corn', 238: 'Dbl Crop WinWht/Cotton', 239: 'Dbl Crop Soybeans/Cotton', 240: 'Dbl Crop Soybeans/Oats', 241: 'Dbl Crop Corn/Soybeans', 242: 'Blueberries', 243: 'Cabbage', 244: 'Cauliflower', 245: 'Celery', 246: 'Radishes', 247: 'Turnips', 248: 'Eggplants', 249: 'Gourds', 250: 'Cranberries', 254: 'Dbl Crop Barley/Soybeans' } @property def non_crop(self): return { 37: 'Other Hay/Non Alfalfa', 59: 'Sod/Grass Seed', 60: 'Switchgrass', 63: 'Forest', 64: 'Shrubland', 65: 'Barren', 81: 'Clouds/No Data', 82: 'Developed', 83: 'Water', 87: 'Wetlands', 88: 'Nonag/Undefined', 92: 'Aquaculture', 111: 'Open Water', 112: 'Perennial Ice/Snow', 121: 'Developed/Open Space', 122: 'Developed/Low Intensity', 123: 'Developed/Med Intensity', 124: 'Developed/High Intensity', 131: 'Barren', 141: 'Deciduous Forest', 142: 'Evergreen Forest', 143: 'Mixed Forest', 152: 'Shrubland', 176: 'Grass/Pasture', 190: 'Woody Wetlands', 195: 'Herbaceous Wetlands' }
def test_raster(self): bb = RasterBounds(raster=self.image) self.assertEqual(bb.get_nwse_tuple(), (47.61118761194992, -111.94851161596767, 47.54916612467683, -111.8361513308256))
def makePolyGeomFromRasterBounds(rastFn): rast = rasterio.open(rastFn) bb = rast.bounds l = bb[0] b = bb[1] r = bb[2] t = bb[3] ul = (l,t) ll = (l,b) ur = (r,t) lr = (r,l) mpGeo = geometry.asPolygon((ul,ur,lr,ll)) bb = RasterBounds(affine_transform=profile['affine'], profile=profile, latlon=True) # open lake centriod shapefile lakeCentShp = fiona.open(lakeCentShpFn) nLake = len(lakeCentShp) # get array of lake centroid coordinates centArr = getShapefileCoordinatesFromMultipoint(lakeCentShpFn)
def __init__(self, obj): ''' :param obj: Directory containing an unzipped Landsat 5, 7, or 8 image. This should include at least a tif for each band, and a .mtl file. ''' self.obj = obj if os.path.isdir(obj): self.isdir = True self.date_acquired = None self.file_list = os.listdir(obj) self.tif_list = [x for x in os.listdir(obj) if x.endswith('.TIF')] self.tif_list.sort() # parse metadata file into attributes # structure: {HEADER: {SUBHEADER: {key(attribute), val(attribute value)}}} self.mtl = mtl.parsemeta(obj) self.meta_header = list(self.mtl)[0] self.super_dict = self.mtl[self.meta_header] for key, val in self.super_dict.items(): for sub_key, sub_val in val.items(): # print(sub_key.lower(), sub_val) setattr(self, sub_key.lower(), sub_val) self.satellite = self.landsat_scene_id[:3] # create numpy nd_array objects for each band self.band_list = [] self.tif_dict = {} for i, tif in enumerate(self.tif_list): raster = os.path.join(self.obj, tif) # set all lower case attributes tif = tif.lower() front_ind = tif.index('b') end_ind = tif.index('.tif') att_string = tif[front_ind:end_ind] self.band_list.append(att_string) self.tif_dict[att_string] = raster self.band_count = i + 1 if i == 0: with rasopen(raster) as src: transform = src.transform profile = src.profile meta = src.meta.copy() self.rasterio_geometry = meta self.profile = profile self.transform = transform self.shape = (1, profile['height'], profile['width']) bounds = RasterBounds(affine_transform=transform, profile=profile, latlon=False) self.bounds = bounds self.north, self.west, self.south, self.east = bounds.get_nwse_tuple( ) self.coords = bounds.as_tuple('nsew') self.solar_zenith = 90. - self.sun_elevation self.solar_zenith_rad = self.solar_zenith * pi / 180 self.sun_elevation_rad = self.sun_elevation * pi / 180 self.earth_sun_dist = self.earth_sun_d(self.date_acquired) dtime = datetime.strptime(str(self.date_acquired), '%Y-%m-%d') julian_day = dtime.strftime('%j') self.doy = int(julian_day) self.scene_coords_deg = self._scene_centroid() self.scene_coords_rad = deg2rad(self.scene_coords_deg[0]), deg2rad( self.scene_coords_deg[1])