def _reproject(self): reproj_path = os.path.join(self.temp_dir, 'reproj.tif') setattr(self, 'reprojection', reproj_path) with rasopen(self.projection, 'r') as src: src_profile = src.profile src_bounds = src.bounds src_array = src.read(1) dst_profile = copy.deepcopy(self.target_profile) dst_profile['dtype'] = float32 bounds = src_bounds dst_affine, dst_width, dst_height = cdt(src_profile['crs'], dst_profile['crs'], src_profile['width'], src_profile['height'], *bounds) dst_profile.update({ 'crs': dst_profile['crs'], 'transform': dst_affine, 'width': dst_width, 'height': dst_height }) with rasopen(reproj_path, 'w', **dst_profile) as dst: dst_array = empty((1, dst_height, dst_width), dtype=float32) reproject(src_array, dst_array, src_transform=src_profile['transform'], src_crs=src_profile['crs'], dst_crs=self.target_profile['crs'], dst_transform=dst_affine, resampling=Resampling.nearest, num_threads=2) dst.write( dst_array.reshape(1, dst_array.shape[1], dst_array.shape[2]))
def save(array, geometry, output_filename, crs=None, return_array=False): try: array = array.reshape(1, array.shape[1], array.shape[2]) except IndexError: array = array.reshape(1, array.shape[0], array.shape[1]) geometry['dtype'] = str(array.dtype) if crs: geometry['crs'] = CRS({'init': crs}) with rasopen(output_filename, 'w', **geometry) as dst: dst.write(array) if return_array: return array return None
def __init__(self, target_profile=None, year=None, out_dir=None, from_file=None): self.url_base = 'https://nassgeodata.gmu.edu/axis2/services/CDLService/' \ 'GetCDLFile?year={year}&bbox={wsen}' if from_file: self.from_file = from_file with rasopen(from_file) as src: self.cdl = src.read() self.target_profile = src.profile self.cdl_empty = False else: self.cdl_empty = True self.cdl = None if not out_dir: self.cdl_location = os.path.join(os.path.dirname(__file__), 'model_data') else: self.cdl_location = out_dir self.zip_file = os.path.join(self.cdl_location, '{}_30m_cdls.zip'.format(year)) self.temp_dir = mkdtemp() if target_profile and year: self.target_profile = target_profile self.bbox = RasterBounds( profile=self.target_profile, affine_transform=self.target_profile['transform']) self.bbox.expand(**{ 'east': 0.1, 'west': -0.1, 'north': 0.2, 'south': -0.2 }) self.bbox_projected = bb = self.bbox.to_lambert_conformal_conic( ) bb_str = '{},{},{},{}'.format(bb[0], bb[1], bb[2], bb[3]) self.request_url = self.url_base.format(year=year, wsen=bb_str) self.data_url = self._get_data_url() self.original_tif = None self.mask = None self.projection = None self.reprojection = None
def _resample(self): resample_path = os.path.join(self.temp_dir, 'resample.tif') with rasopen(self.mask, 'r') as src: array = src.read(1) profile = src.profile res = src.res try: target_affine = self.target_profile['affine'] except KeyError: target_affine = self.target_profile['transform'] target_res = target_affine.a res_coeff = res[0] / target_res new_array = empty(shape=(1, round(array.shape[0] * res_coeff), round(array.shape[1] * res_coeff)), dtype=float32) aff = src.transform new_affine = Affine(aff.a / res_coeff, aff.b, aff.c, aff.d, aff.e / res_coeff, aff.f) profile['transform'] = self.target_profile['transform'] profile['width'] = self.target_profile['width'] profile['height'] = self.target_profile['height'] profile['dtype'] = str(new_array.dtype) delattr(self, 'mask') with rasopen(resample_path, 'w', **profile) as dst: reproject(array, new_array, src_transform=aff, dst_transform=new_affine, src_crs=src.crs, dst_crs=src.crs, resampling=Resampling.nearest) dst.write(new_array) with rasopen(resample_path, 'r') as src: arr = src.read() return arr
def setUp(self): self.box = (-109.9849, 46.46738, -109.93647, 46.498625) self.dst_srs = '26912' self.kwargs = dict([('dst_crs', self.dst_srs)]) self.kwargs_centroid = dict([('dst_crs', self.dst_srs), ('centroid', ((self.box[1] + self.box[3]) / 2, ((self.box[0] + self.box[2]) / 2))), ('buffer', 1700)]) self.tile_loc = os.path.join(os.path.dirname(__file__), 'data', 'wheatland_tile.tif') with rasopen(self.tile_loc, 'r') as src: self.profile = src.profile self.array = src.read()
def save(self, array, geometry, output_filename, crs=None): array = array.reshape(geometry['count'], array.shape[1], array.shape[2]) geometry['dtype'] = uint8 if crs: dst_crs = CRS({'init': 'epsg:{}'.format(crs)}) if geometry['crs'] != dst_crs: self.reproject_multiband(output_filename, dst_crs) return None with rasopen(output_filename, 'w', **geometry) as dst: dst.write(array) return None
def mask_raster_to_features(raster, features, features_meta): # This function is useful when you don't have access to the # file from which the features came or if the file doesn't exist. gdf = gpd.GeoDataFrame.from_features(features, features_meta) # do I need # the whole metadata? gdf = gdf[gdf.geometry.notnull()] with rasopen(raster, 'r') as src: crs = CRS(src.crs['init']) print(crs) shp = gdf.to_crs(src.crs) features = get_features(shp) arr = src.read() out_image, out_transform = mask(src, shapes=features) out_image[out_image != 0] = 1 meta = src.meta return out_image, meta
def test_conforming_array(self): """ Test shape of Gridmet vs. Landsat image. :return: """ l8 = Landsat8(self.dir_name_LC8) shape = 1, l8.rasterio_geometry['height'], l8.rasterio_geometry[ 'width'] polygon = l8.get_tile_geometry() cdl = Cdl(year=self.year, target_profile=l8.profile, out_dir=self.dir_name_LC8) _ = cdl.get_conforming_data(polygon) with rasopen(os.path.join(self.dir_name_LC8, 'cdl.tif')) as dst: arr = dst.read() self.assertEqual(arr.shape, shape)
def _point_raster_extract(self, raster, _name): with rasopen(raster, 'r') as rsrc: rass_arr = rsrc.read() rass_arr = rass_arr.reshape(rass_arr.shape[1], rass_arr.shape[2]) affine = rsrc.transform s = Series(index=range(0, self.extracted_points.shape[0]), name=_name) for ind, row in self.extracted_points.iterrows(): x, y = self._geo_point_to_projected_coords(row['X'], row['Y']) c, r = ~affine * (x, y) try: raster_val = rass_arr[int(r), int(c)] s[ind] = float(raster_val) except IndexError: s[ind] = None return s
def clip_raster(evaluated, path, row, outfile=None): out = _get_path_row_geometry(path, row) with rasopen(evaluated, 'r') as src: out = out.to_crs(src.crs['init']) features = get_features(out) # if crop == true for mask, you have to update the metadata. out_image, out_transform = mask(src, shapes=features, crop=True, nodata=np.nan) meta = src.meta.copy() count = out_image.shape[0] meta.update({"driver": "GTiff", "height": out_image.shape[1], "width": out_image.shape[2], "transform": out_transform}) if outfile is not None: save_raster(out_image, outfile, meta, count)
def test_warped_vrt(self): warped_vrt.warp_vrt(self.directory) shapes = [] dirs = [ _ for _ in os.listdir(self.directory) if not _.endswith('.txt') ] for d in dirs: lst = [ _ for _ in os.listdir(os.path.join(self.directory, d)) if _.endswith('.TIF') ] for l in lst: tif = os.path.join(self.directory, d, l) with rasopen(tif, 'r') as src: shapes.append(src.shape) shutil.rmtree(self.directory) self.assertEqual(shapes[0], shapes[1])
def write_raster(self, out_file, new_array=None): if isinstance(new_array, ndarray): self.new_array = new_array try: self.new_array = self.new_array.reshape(1, self.new_array.shape[1], self.new_array.shape[2]) except IndexError: self.new_array = self.new_array.reshape(1, self.new_array.shape[0], self.new_array.shape[1]) self.raster_geo['dtype'] = str(self.new_array.dtype) self.raster_geo['count'] = 1 with rasopen(out_file, 'w', **self.raster_geo) as dst: dst.write(self.new_array) return None
def get_image(self, state): """ Get NAIP imagery from states excluding Hawaii and Alaska Current hack in this method and in GeoBounds is hard-coded epsg: 3857 'web mercator', though the NAIP service provides epsg: 102100 a deprecated ESRI SRS' :param state: e.g. 'ND' :param size: tuple of horizontal by vertical size in pixels, e.g., (512, 512) :return: """ coords = { x: y for x, y in zip(['west', 'south', 'east', 'north'], self.bbox) } w, s, e, n = GeoBounds(**coords).to_web_mercator() self.web_mercator_bounds = (w, s, e, n) bbox_str = self.bounds_fmt.format(w=w, s=s, e=e, n=n) naip_str = '{}_NAIP'.format(state) query = self.usda_query_str.format(naip_str, bbox_str) url = '{}{}'.format(self.naip_base_url, query) req = get(url, verify=False, stream=True) if req.status_code != 200: raise ValueError('Bad response {} from NAIP API request.'.format( req.status_code)) # with open(self.temp_file, 'wb') as f: # f.write(req.content) with open(self.temp_file, 'wb') as f: for chunk in req.iter_content(chunk_size=1024): if chunk: f.write(chunk) with rasopen(self.temp_file, 'r') as src: array = src.read() profile = src.profile return array, profile
def mask_raster_to_shapefile(shapefile, raster, return_binary=True): ''' Generates a mask with 1 everywhere shapefile data is present and a no_data value everywhere else. no_data is -1 in this case, as it is never a valid class label. Switching coordinate reference systems is important here, or else the masking won't work. ''' shp = gpd.read_file(shapefile) shp = shp[shp.geometry.notnull()] with rasopen(raster, 'r') as src: # pyproj deprecated the +init syntax. crs = CRS(src.crs['init']) shp = shp.to_crs(crs) features = get_features(shp) arr = src.read() out_image, out_transform = mask(src, shapes=features, filled=False) if return_binary: out_image[out_image != 0] = 1 meta = src.meta return out_image, meta
def __init__(self, raster=None, affine_transform=None, profile=None, latlon=True): BBox.__init__(self) if raster: with rasopen(raster, 'r') as src: profile = src.profile affine = profile['transform'] if affine_transform: affine = affine_transform col, row = 0, 0 w, n = affine * (col, row) col, row = profile['width'], profile['height'] e, s = affine * (col, row) if latlon and profile['crs'] != CRS({'init': 'epsg:4326'}): in_proj = Proj(init=profile['crs']['init']) self.west, self.north = in_proj(w, n, inverse=True) self.east, self.south = in_proj(e, s, inverse=True) else: self.north, self.west, self.south, self.east = n, w, s, e
def get_mask(self, clip_geometry=None, out_file=None): arr = None if self.cdl_empty: try: arr = self.get_conforming_data(clip_geometry=clip_geometry) except ValueError: print('Need clip geometry to build cdl') else: arr = self.cdl crop = list(self.crop.keys()) msk = isin(arr, crop) msk = ~msk msk = msk.astype(uint8) profile = copy.deepcopy(self.target_profile) profile['dtype'] = uint8 if out_file: with rasopen(out_file, 'w', **profile) as dst: dst.write(msk) return msk
def iterate_over_image_and_evaluate_patchwise_lstm_cnn(image_stack, model_path, out_filename, out_meta, n_classes, tile_size=24): model = load_model(model_path, custom_objects={'m_acc': m_acc}) timeseries = [] for i in range(0, image_stack.shape[0] - 3, 3): timeseries.append(image_stack[i:i + 3]) timeseries = np.asarray(timeseries) timeseries = np.swapaxes(timeseries, 1, 3) timeseries = np.expand_dims(timeseries, 0) print(timeseries.shape) for start_idx in range(0, timeseries.shape[1] - 12): predictions = np.zeros( (timeseries.shape[2], timeseries.shape[3], n_classes)) timeseries_copy = timeseries[:, start_idx:start_idx + 12, :, :, :] for i in range(0, timeseries_copy.shape[2] - tile_size, tile_size): for j in range(0, timeseries_copy.shape[3] - tile_size, tile_size): image_tile = timeseries_copy[:, :, i:i + tile_size, j:j + tile_size, :] if np.all(image_tile == 0): continue preds = np.squeeze(model.predict(image_tile)) predictions[i:i + tile_size, j:j + tile_size, :] = np.sum(preds, axis=0) stdout.write("{}, {:.3f}\r".format(start_idx, i / timeseries_copy.shape[2])) predictions = np.swapaxes(predictions, 0, 2) out_meta.update({'count': n_classes, 'dtype': np.float64}) with rasopen(out_filename, "w", **out_meta) as dst: dst.write(predictions)
def reproject_tiles(self): reproj_path = os.path.join(self.temp_dir, 'tiled_reproj.tif') setattr(self, 'reprojection', reproj_path) profile = copy.deepcopy(self.target_profile) profile['dtype'] = float32 bb = self.web_mercator_bounds bounds = (bb[0], bb[1], bb[2], bb[3]) dst_affine, dst_width, dst_height = calculate_default_transform( self.merged_profile['crs'], profile['crs'], self.merged_profile['width'], self.merged_profile['height'], *bounds) profile.update({ 'crs': profile['crs'], 'transform': dst_affine, 'width': dst_width, 'height': dst_height }) with rasopen(reproj_path, 'w', **profile) as dst: dst_array = empty((1, dst_height, dst_width), dtype=float32) reproject(self.merged_array, dst_array, src_transform=self.merged_transform, src_crs=self.merged_profile['crs'], dst_crs=self.target_profile['crs'], dst_transform=dst_affine, resampling=Resampling.cubic, num_threads=2) dst.write( dst_array.reshape(1, dst_array.shape[1], dst_array.shape[2])) delattr(self, 'merged_array')
def stack_images_from_list_of_filenames_sorted_by_date(filenames): filenames = sorted(filenames, key=lambda x: parse_date(x)) dates = [parse_date(x) for x in filenames] # if len(filenames) > 16: # filenames = filenames[:16] first = True image_stack = None i = 0 n_bands = 7 if not len(filenames): print('empty list of filenames') return (None, None, None, None) for filename in filenames: with rasopen(filename, 'r') as src: arr = src.read() meta = deepcopy(src.meta) if first: first = False image_stack = np.zeros((n_bands * len(filenames) + len(filenames), arr.shape[1], arr.shape[2]), dtype=np.int16) target_meta = deepcopy(meta) target_fname = filename image_stack[0:n_bands] = arr i += n_bands else: try: image_stack[i:i + n_bands] = arr i += n_bands except ValueError as e: arr = warp_single_image(filename, target_meta) image_stack[i:i + n_bands] = arr i += n_bands image_stack[-len(filenames):] = date_stack(dates, image_stack.shape) return image_stack, target_meta, target_fname, meta
def save_raster(arr, outfile, meta, count=5): meta.update(count=count) with rasopen(outfile, 'w', **meta) as dst: dst.write(arr)
def _get_mask_from_raster(self, extra_mask): with rasopen(extra_mask, mode='r') as src: arr = src.read() self.raster_geo = src.meta.copy() return arr
def load_raster(raster_name): with rasopen(raster_name, 'r') as src: arr = src.read() meta = src.meta.copy() return arr, meta
def _load_image(f, image=False): with rasopen(f, 'r') as src: im = src.read() return im
def _get_crs(self): for key, val in self.paths_map.items(): with rasopen(val, 'r') as src: crs = src.crs break return crs
from rasterio import open as rasopen import matplotlib.pyplot as plt import numpy as np import sys raster = sys.argv[1] with rasopen(raster) as src: arr = np.squeeze(src.read()) src = None print(arr.shape) print(arr.dtype) arr[arr == -9999] = np.nan ts = 5000 for i in range(0, arr.shape[0] - ts, ts): for j in range(0, arr.shape[1] - ts, ts): plt.imshow(arr[i:i + ts, j:j + ts]) plt.colorbar() plt.show()
def warp_vrt(directory, delete_extra=False, use_band_map=False, overwrite=False, remove_bqa=True): """ Read in image geometry, resample subsequent images to same grid. The purpose of this function is to snap many Landsat images to one geometry. Use Landsat578 to download and unzip them, then run them through this to get identical geometries for analysis. Files :param use_band_map: :param delete_extra: :param directory: A directory containing sub-directories of Landsat images. :return: None """ if 'resample_meta.txt' in os.listdir(directory) and not overwrite: print('{} has already had component images warped'.format(directory)) return None mapping = {'LC8': Landsat8, 'LE7': Landsat7, 'LT5': Landsat5} vrt_options = {} list_dir = [ x[0] for x in os.walk(directory) if os.path.basename(x[0])[:3] in mapping.keys() ] extras = [ os.path.join(directory, x) for x in os.listdir(directory) if x.endswith('.tif') ] first = True for d in list_dir: sat = LandsatImage(d).satellite paths = extras root = os.path.join(directory, d) if os.path.isdir(root): for x in os.listdir(root): if remove_bqa and x.endswith('BQA.TIF'): try: os.remove(x) except FileNotFoundError: pass elif use_band_map: bands = BandMap().selected for y in bands[sat]: if x.endswith('B{}.TIF'.format(y)): paths.append(os.path.join(directory, d, x)) else: if x.endswith('.TIF') or x.endswith('.tif'): paths.append(os.path.join(directory, d, x)) if x.endswith('MTL.txt'): mtl = os.path.join(directory, d, x) if first: landsat = mapping[sat](os.path.join(directory, d)) dst = landsat.rasterio_geometry vrt_options = { 'resampling': Resampling.nearest, 'dst_crs': dst['crs'], 'dst_transform': dst['transform'], 'dst_height': dst['height'], 'dst_width': dst['width'] } message = """ This directory has been resampled to same grid. Master grid is {}. {} """.format(d, datetime.now()) with open(os.path.join(directory, 'resample_meta.txt'), 'w') as f: f.write(message) first = False os.rename(mtl, mtl.replace('.txt', 'copy.txt')) for tif_path in paths: print('warping {}'.format(os.path.basename(tif_path))) with rasopen(tif_path, 'r') as src: with WarpedVRT(src, **vrt_options) as vrt: data = vrt.read() dst_dir, name = os.path.split(tif_path) outfile = os.path.join(dst_dir, name) meta = vrt.meta.copy() meta['driver'] = 'GTiff' with rasopen(outfile, 'w', **meta) as dst: dst.write(data) os.rename(mtl.replace('.txt', 'copy.txt'), mtl) if delete_extra: for x in os.listdir(os.path.join(directory, d)): x_file = os.path.join(directory, d, x) if x_file not in paths: if x[-7:] not in ['ask.tif', 'MTL.txt']: print('removing {}'.format(x_file)) os.remove(x_file)
def point_target_extract(points, nlcd_path, target_shapefile=None, count_limit=None): point_data = {} with fopen(points, 'r') as src: for feature in src: name = feature['id'] proj_coords = feature['geometry']['coordinates'] point_data[name] = {'point': feature['geometry'], 'coords': proj_coords} # point_crs = src.profile['crs']['init'] pt_ct = 0 for pt_id, val in point_data.items(): pt_ct += 1 if pt_ct < count_limit: pt = shape(val['point']) with fopen(target_shapefile, 'r') as target_src: has_attr = False for t_feature in target_src: polygon = t_feature['geometry'] if pt.within(shape(polygon)): print('pt id {}, props: {}' .format(pt_id, t_feature['properties'])) props = t_feature['properties'] point_data[pt_id]['properties'] = {'IType': props['IType'], 'LType': props['LType']} has_attr = True break if not has_attr: if nlcd_path: with rasopen(nlcd_path, 'r') as rsrc: rass_arr = rsrc.read() rass_arr = rass_arr.reshape(rass_arr.shape[1], rass_arr.shape[2]) affine = rsrc.affine x, y = val['coords'] col, row = ~affine * (x, y) raster_val = rass_arr[int(row), int(col)] ltype_dct = {'IType': None, 'LType': str(raster_val)} point_data[pt_id]['properties'] = ltype_dct print('id {} has no FLU, ' 'nlcd {}'.format(pt_id, nlcd_value(ltype_dct['LType']))) else: ltype_dct = {'IType': None, 'LType': None} point_data[pt_id]['properties'] = ltype_dct idd = [] ltype = [] itype = [] x = [] y = [] ct = 0 for pt_id, val in point_data.items(): ct += 1 if ct < count_limit: idd.append(pt_id) ltype.append(val['properties']['LType']) itype.append(val['properties']['IType']) x.append(val['coords'][0]) y.append(val['coords'][1]) else: break dct = dict(zip(['ID', 'LTYPE', 'ITYPE', 'X', 'Y'], [idd, ltype, itype, x, y])) df = DataFrame(data=dct) return df
def __init__(self, obj): ''' :param obj: Directory containing an unzipped Landsat 5, 7, or 8 image. This should include at least a tif for each band, and a .mtl file. ''' self.obj = obj if os.path.isdir(obj): self.isdir = True self.date_acquired = None self.file_list = os.listdir(obj) self.tif_list = [x for x in os.listdir(obj) if x.endswith('.TIF')] self.tif_list.sort() # parse metadata file into attributes # structure: {HEADER: {SUBHEADER: {key(attribute), val(attribute value)}}} self.mtl = mtl.parsemeta(obj) self.meta_header = list(self.mtl)[0] self.super_dict = self.mtl[self.meta_header] for key, val in self.super_dict.items(): for sub_key, sub_val in val.items(): # print(sub_key.lower(), sub_val) setattr(self, sub_key.lower(), sub_val) self.satellite = self.landsat_scene_id[:3] # create numpy nd_array objects for each band self.band_list = [] self.tif_dict = {} for i, tif in enumerate(self.tif_list): raster = os.path.join(self.obj, tif) # set all lower case attributes tif = tif.lower() front_ind = tif.index('b') end_ind = tif.index('.tif') att_string = tif[front_ind:end_ind] self.band_list.append(att_string) self.tif_dict[att_string] = raster self.band_count = i + 1 if i == 0: with rasopen(raster) as src: transform = src.transform profile = src.profile meta = src.meta.copy() self.rasterio_geometry = meta self.profile = profile self.transform = transform self.shape = (1, profile['height'], profile['width']) bounds = RasterBounds(affine_transform=transform, profile=profile, latlon=False) self.bounds = bounds self.north, self.west, self.south, self.east = bounds.get_nwse_tuple( ) self.coords = bounds.as_tuple('nsew') self.solar_zenith = 90. - self.sun_elevation self.solar_zenith_rad = self.solar_zenith * pi / 180 self.sun_elevation_rad = self.sun_elevation * pi / 180 self.earth_sun_dist = self.earth_sun_d(self.date_acquired) dtime = datetime.strptime(str(self.date_acquired), '%Y-%m-%d') julian_day = dtime.strftime('%j') self.doy = int(julian_day) self.scene_coords_deg = self._scene_centroid() self.scene_coords_rad = deg2rad(self.scene_coords_deg[0]), deg2rad( self.scene_coords_deg[1])