def ExtractPointValues(raster, shp):
""" Extract raster values by a shapefile point mask.
"""
import shapefile, rasterio
from rasterstats import point_query
import numpy as np
# Raster management
r = rasterio.open(raster)
affine = r.affine
bands = r.count
bandNames = []
for i in range(bands):
a = "B" + str(i+1)
bandNames.append(a)
# empty matrix to store results
matrix = np.empty((len(shapefile.Reader(shp).records()), bands))
# Extract values
for i in range(bands):
# stats
array = r.read(i+1) # open one band at a time
stats = point_query(shp, array, affine=affine)
matrix[:,i] = stats
return matrix
def getconcen(act_num, rasterfile, df, routegeom, ext = 100, extgaus = 2000, sd=300, indoor_ratio = 0.7 ):
ph, pw = gethw(df)
return {
1: indoor_ratio * point_query(ph, rasterfile ).pop(), #home
2: np.nan_to_num(np.mean(point_query(routegeom, rasterfile)),0),# route #can easily make buffer out of it as well, here the ap already 100m so not needed.
3: indoor_ratio *point_query(pw, rasterfile ).pop(), # work_indoor
4: buffermean(p, ext, rasterfile), # freetime 1 will change into to sport (second route)
5: gaussKernconv(p, extgaus, rasterfile, sd = 1), # freetime 2, distance decay, outdoor.
6: buffermean(p, ext, rasterfile) # freetime 3, in garden or terras
} [act_num]
def create_final_od_grid(df,height_div):
height = numpy.sqrt(pygeos.area(df.geometry)/height_div).values[0]
grid = pd.DataFrame(create_grid(create_bbox(df),height),columns=['geometry'])
#clip grid of bbox to grid of the actual spatial exterior of the country
clip_grid = pygeos.intersection(grid,df.geometry)
clip_grid = clip_grid.loc[~pygeos.is_empty(clip_grid.geometry)]
# turn to shapely geometries again for zonal stats
clip_grid.geometry = pygeos.to_wkb(clip_grid.geometry)
clip_grid.geometry = clip_grid.geometry.apply(loads)
clip_grid = gpd.GeoDataFrame(clip_grid)
# get total population per grid cell
if height < 0.01:
clip_grid['tot_pop'] = clip_grid.geometry.apply(lambda x: point_query(x,world_pop))
clip_grid['tot_pop'] = clip_grid['tot_pop'].apply(lambda x: np.sum(x))
else:
clip_grid['tot_pop'] = clip_grid.geometry.apply(lambda x: zonal_stats(x,world_pop,stats="sum"))
clip_grid['tot_pop'] = clip_grid['tot_pop'].apply(lambda x: x[0]['sum'])
# remove cells in the grid that have no population data
clip_grid = clip_grid.loc[~pd.isna(clip_grid.tot_pop)]
if len(clip_grid) > 100:
clip_grid = clip_grid.loc[clip_grid.tot_pop > 100]
clip_grid.reset_index(inplace=True,drop=True)
clip_grid.geometry = clip_grid.geometry.centroid
clip_grid['GID_0'] = country_network[:3]
clip_grid['grid_height'] = height
#print(len(clip_grid),height)
return clip_grid
def getProfileChart(product, dates, length, resolution, line):
## Get coordinates along line to get the values at.
coordinates = getCoordinatesAlongLine(line[0], line[1], resolution)
productLocation = getProductLocation(product)
productLocation = f'{productLocation}/{dates}/{product}'
rasters = os.listdir(productLocation)
for j in rasters:
if j.endswith('vrt'):
raster = j
timeseries = {"datasets": []}
data = []
i = 0
for point in coordinates:
pt = Point(point[0], point[1])
try:
productValue = point_query([pt], f'{productLocation}/{raster}')
productValue = np.round(productValue, 2)[0]
except TypeError as e:
print(e)
productValue = None
data.append({"x": i, "y": productValue})
i += 1
timeseries["datasets"].append({
"fid": 0,
"label": f'{product}, {dates}',
"data": data
})
return timeseries
def do_point_query(points, raster, csv_pth, num, log_out):
try:
u.verify_dir(csv_pth)
dataset = raster.split('\\')[-2].replace(' ', '_')
raster_name_to_csv = os.path.basename(raster).replace('.tif', '.csv')
csv_name = '__'.join([dataset, raster_name_to_csv])
csv_out = os.path.join(csv_pth, csv_name)
start = timer()
u.write_to_log(' {}) Raster: {}'.format(num, os.path.basename(raster)), log_out)
stats = point_query(points, raster, interpolate='nearest', geojson_out=True)
print(' point_query... ({} sec.)'.format(round(timer()-start, 2)))
start = timer()
attributes = []
for item in stats:
#print ('{}'.format(item['properties']))
attributes.append(item['properties'])
print(' append dicts... ({} sec.)'.format(round(timer()-start, 2)))
start = timer()
with open(csv_out, 'w', newline='') as outfile:
fp = csv.DictWriter(outfile, attributes[0].keys())
fp.writeheader()
fp.writerows(attributes)
print(' write to csv... ({} sec.)'.format(round(timer()-start, 2)))
u.write_to_log(' CSV file: {}'.format(csv_out), log_out)
u.write_to_log(' Log file: {}'.format(log_out), log_out)
except Exception as e:
u.write_to_log(str(e), log_out)
u.write_to_log('FINISH POINTS: {}'.format(time.strftime("%Y-%m-%d %H:%M:%S")), log_out)
def ExtractPointValues(raster, shp):
""" Extract raster values by a shapefile point mask.
"""
import shapefile, rasterio
from rasterstats import point_query
import numpy as np
# Raster management
with rasterio.open(raster) as r:
affine = r.affine
bands = r.count
bandNames = []
for i in range(bands):
a = "B" + str(i + 1)
bandNames.append(a)
# empty matrix to store results
matrix = np.empty((len(shapefile.Reader(shp).records()), bands))
# Extract values
for i in range(bands):
# stats
array = r.read(i + 1) # open one band at a time
stats = point_query(shp, array, affine=affine)
matrix[:, i] = stats
return matrix
def run_function(params):
# Prepare the parameters
subject_ref = params['subject']
raster = params['raster']
stats = params.get('stats', 'mean')
geojson_out = params.get('geojson_out', True)
try:
subject = fiona.open(subject_ref)
subject_types = get_subject_types(subject)
if 'Polygon' in subject_types or 'MultiPolygon' in subject_types:
features = zonal_stats(subject, raster, stats=stats, geojson_out=geojson_out)
elif 'Point' in subject_types or 'MultiPoint' in subject_types:
features = point_query(subject, raster, geojson_out=geojson_out)
else:
raise ValueError("Input features need to be either all Polygons or Points. Doesn't look like they are.")
# Decorate return as either GeoJSON FeatureCollection (other way to do this?) or just the array of stats
if geojson_out:
return FeatureCollection(features)
else:
return features
except AttributeError:
raise ValueError("Error calculating zonalstats. Please confirm every input "
"geometry is valid, and contains coordinates")
def point_query(df, path, crs, *args, **kwargs):
if not df.crs == crs:
df = df.to_crs(crs)
return rstats.point_query(df.geometry,
path,
interpolate=interpolate,
*args,
**kwargs)
def eval_raster(df_data, raster_file, min_prec=1., factor=1.):
"""
Evaluates a raster based on point information.
:param df_data: DataFrame with point information.
:type df_data: pd.DataFrame
:param raster_file: path/filename for raster image.
:type raster_file: str
:param min_prec: Minimum value for evaluating precipitation.
:type min_prec: float
:param factor: Factor for multiplying the raster.
:type factor: float
:return: No Rain Success, Rain Success and Mean Absolute Error (MAE)
:rtype: dict
"""
geometry = [
Point(xy) for xy in zip(df_data['Longitude'], df_data['Latitude'])
]
df = df_data.drop(['Longitude', 'Latitude'], axis=1)
crs = {'init': 'epsg:4326'}
gdf = gpd.GeoDataFrame(df, crs=crs, geometry=geometry)
results = point_query(gdf,
raster_file,
property_name='Goes',
geojson_out=True)
gdf_stats = gpd.GeoDataFrame.from_features(results)
gdf_stats.loc[gdf_stats['Goes'] < min_prec, 'Goes'] = 0
gdf_stats['Goes'] = gdf_stats['Goes'] * factor
gdf_stats.loc[gdf_stats['Value'] < min_prec, 'Value'] = 0
idx_norain = gdf_stats[(gdf_stats['Value'] == 0)
& (gdf_stats['Goes'] == 0)].index
idx_rain = gdf_stats[(gdf_stats['Value'] > 0)
& (gdf_stats['Goes'] > 0)].index
try:
succ_norain = len(idx_norain) / float(
len(gdf_stats[(gdf_stats['Value'] == 0)]))
except ZeroDivisionError:
succ_norain = None
try:
succ_rain = len(idx_rain) / float(
len(gdf_stats[(gdf_stats['Value'] > 0)]))
except ZeroDivisionError:
succ_rain = None
sr_mae = gdf_stats.loc[idx_rain, 'Value'] - gdf_stats.loc[idx_rain, 'Goes']
mae = sr_mae.abs().mean()
return {
'No Rain Success': succ_norain,
'Rain Success': succ_rain,
'Rain MAE': mae
}
def find_grid_duplicates(
gdf: gpd.GeoDataFrame,
species_col: str,
resolution: float,
bounds: Union[list, tuple] = None,
keep: Union[bool, str] = False,
) -> pd.Series:
"""
Find records of the same species that are in the same cell of a
specific grid.
Parameters
----------
gdf : GeoDataFrame
GeoDataFrame with records.
species_col : str
Column name with the species name for each record.
resolution : float
Grid resolution.
bounds : list or tuple
Grid bounds (xmin, ym, xmax, ymax). If no bounds are passed, the
bounds from gdf will be taken.
keep : str
Which duplicates to mark. Can be:
- False: mark all duplicates as True.
- 'first': mark duplicates as True except for the first occurrence.
- 'last': mark duplicates as True except for the last occurrence.
Returns
-------
pd.Series
Boolean Series indicating whether records are spatial duplicates.
Notes
-----
bounds and resolution should match gdf coordinate reference
system.
"""
gdf = gdf.copy()
if not bounds:
bounds = gdf.geometry.total_bounds
grid = _create_id_grid(*bounds, resolution, gdf.crs.srs)
gdf["__grid_id"] = rasterstats.point_query(gdf,
grid.read(1),
affine=grid.transform,
interpolate="nearest",
nodata=-9999)
result = gdf.duplicated(subset=[species_col, "__grid_id"], keep=keep)
# Result for records that do not have a grid ID is left empty.
no_grid_id = gdf["__grid_id"].isna()
result.loc[no_grid_id] = np.nan
return result
def create_3d_pnt(pnt, dem):
'''Creates a z-enabled point from a 2d pnt and elevation data.'''
z = point_query(pnt, dem)[0]
try:
x, y, _ = list(pnt.coords)[0]
except:
print("2d point provided")
x, y = list(pnt.coords)[0]
return Point([x, y, z])
def test_point_query_nodata():
# all nodata, on the grid
point = "POINT(245309 1000308)"
val = point_query(point, raster_nodata)[0]
assert val is None
# all nodata, off the grid
point = "POINT(244000 1000308)"
val = point_query(point, raster_nodata)[0]
assert val is None
point = "POINT(244000 1000308)"
val = point_query(point, raster_nodata, interpolate="nearest")[0]
assert val is None
# some nodata, should fall back to nearest
point = "POINT(245905 1000361)"
val = point_query(point, raster_nodata, interpolate="nearest")[0]
assert round(val) == 43
val = point_query(point, raster_nodata)[0]
assert round(val) == 43
def create_topographic_profiles(self, column):
"""Updates 2d lines to 3d topographic profiles"""
lns = self._offset_lines
for i in lns.index:
profile_points = point_query(lns[column][i], dem)[0]
points3d = []
for ii, z in enumerate(profile_points):
x, y = list(lns[column][i].coords)[ii]
points3d.append(Point([x, y, z]))
lns.loc[i][column] = LineString(points3d)
return lns
def pointGtiff(pt, gt, b):
stats = (extractDate(gt),)
bl = b.split(',')
for i in range(len(bl)):
if bl[i].isdigit():
bn = int(bl[i])
else:
#bn = bl[i]
bn = i + 1
stats += (point_query(pt, gt, band=bn)[0], )
#stats = (extractDate(gt), point_query(pt, gt, band=b)[0])
return [stats]
def terrain_p2p(dem, line):
"""
This module takes a set of point coordinates and returns
the surface profile.
Parameters
----------
dem_folder : string
Folder path to the available Digital Elevation Model tiles.
line : dict
Geojson linestring. Must be in WGS84 / EPSG: 4326
current_crs : string
The current coordinate reference system.
Returns
-------
surface_profile :list
Contains the surface profile measurements in meters.
distance_km : float
Distance in kilometers between the antenna and receiver.
points : list of dicts
Location of geojson sampling points.
"""
line_geom = LineString(line['geometry']['coordinates'])
# Geographic distance
geod = pyproj.Geod(ellps="WGS84")
distance_m = geod.geometry_length(line_geom)
distance_km = distance_m / 1e3
# Interpolate along line to get sampling points
num_samples = determine_num_samples(distance_m)
steps = np.interp(range(num_samples), [0,num_samples], [0, line_geom.length])
point_geoms = [line_geom.interpolate(currentdistance) for currentdistance in steps]
# Sample elevation profile
surface_profile = point_query(point_geoms, dem)
# Put together point features with raster values
points = [
{
'type': 'Feature',
'geometry': mapping(point),
'properties': {
'elevation': float(z),
}
}
for point, z in zip(point_geoms, surface_profile)
]
return surface_profile, distance_km, points
def pointNetcdf(pt, nc, v):
ncf = Dataset(nc)
# affine transform(px width, row rotation, UL x-coord, col rotation, px height, UL y-coord)
affine = Affine(0.1, 0.0, -180.0, 0.0, -0.1, 90.0)
ndims = ncf[v].ndim
if ndims == 2:
array = ncf[v][:,:]
ncf.close()
stats = (extractDate(nc), point_query(pt, array, affine=affine, nodata=-9999.0)[0])
return [stats]
elif ndims == 3: # assume it's time/lat/lon
#print 'N-dims:', ndims
cdftime = utime(ncf['time'].units)
stats = []
for i in range(ncf[v].shape[0]):
array = ncf[v][i,:,:]
dt = cdftime.num2date(ncf['time'][i]).strftime('%Y-%m-%d')
stats.append((dt, point_query(pt, array, affine=affine, nodata=-9999.0)[0]))
ncf.close()
#print stats
return stats
else:
print 'Don\'t know what to do with %s dimensions!' % ndims
sys.exit()
def gradient(route_shp, rasterfile):
"""
Calculates the elevation and road grade at each point along the route.
Parameters
----------
route_shp: GeoDataFrame for the selected route;
output of read_shape().
rasterfile: elevation data file (.tif)
Returns
-------
elevation_meters: the elevation at each point along the route
route_gradient: the road grade (e.g. slope) at each point along the route
route_cum_distance: the total route distance at each point along the route [m]
route_distance: the distance between each point [m]
"""
# from the 'rasterstats' userguide:
# "rasterstats, exists solely to extract information from
# geospatial raster data based on vector geometries"
# 'point_query' returns the values defined in the 'rasterfild'
# at the points defined within the GeoDataFrame 'route_shp'
elevation = rasterstats.point_query(route_shp, rasterfile)
# Convert elevations to meters
elevation_meters = np.asarray(elevation) * 0.3048
# Calculate geodesic distances between points as well as cumulative
# distances along route.
route_distance, route_cum_distance = distance_measure(route_shp)
# Calculate route gradient at each point along route
#HJG: I am confused as to why we add zero to the beggining of the
# list here, and also why we take the abs value. Seems like
# the sign of the gradient would determine if we the bus is
# going up or down a hill. I do see that we get back the right
# length list
route_gradient = np.insert(
(np.diff(elevation) / route_distance),
0,
0,
)
return elevation_meters, route_gradient, route_cum_distance, route_distance
def rasterstats_on_geometry(geoframe,
raster_path=None,
raster_dtype='int',
output_colm='mean_value',
stats_type='point_query',
metrics=['mean'],
interpolate='bilinear',
nodata=0,
all_touched=False):
"""
A simple function for the standard procedure to map values from a raster on
a list of geometries in a geodataframe
#currently, caterogical data is not integrated
"""
#correct metrics if input is str
if isinstance(metrics, str):
metrics = [metrics]
#read rasterfile
with rasterio.open(raster_path) as src:
affine = src.transform
if raster_dtype == 'int':
data = np.int32(src.read(1))
else:
data = np.float64(src.read(1))
#depending on the chosen stats type we apply different methods from raster
#stats library
if stats_type == 'point_query':
geoframe[output_colm] = point_query(geoframe.centroid,
data,
affine=affine,
nodata=nodata,
interpolate=interpolate)
if stats_type == 'zonal_stats':
statistic_dict = zonal_stats(geoframe,
data,
stats=metrics,
affine=affine,
nodata=nodata,
all_touched=all_touched)
#we loop through the keys
for key in statistic_dict[0].keys():
geoframe[output_colm + '_' + key] = [
statistic_dict_entry[key]
for statistic_dict_entry in statistic_dict
]
#return geotdataframe
return geoframe
def load_training_vector(response_shapes,
explanatory_rasters,
response_field,
metric='mean'):
"""
Parameters
----------
response_shapes : Source of vector features for raster_stats;
can be OGR file path or iterable of geojson-like features
response_field : Field name containing the known response category (must be numeric)
explanatory_rasters : List of Paths to GDAL rasters containing explanatory variables
metric : Statistic to aggregate explanatory data across line and polygon vector features
Defaults to 'mean' (optional)
Returns
-------
train_xs : Array of explanatory variables
train_y : 1xN array of known responses
"""
from rasterstats import zonal_stats, point_query
all_means = []
all_zones = None
for i, raster in enumerate(explanatory_rasters):
logger.debug("Rasters stats on %s" % raster)
stats = point_query(response_shapes, raster, geojson_out=True)
#point_query(response_shapes, raster, stats=metric, prefix="pyimpute_", geojson_out=True)
#print(stats)
zones = [x['properties'][response_field] for x in stats]
if all_zones:
assert zones == all_zones
else:
all_zones = zones
means = [x['properties']['value'] for x in stats]
all_means.append(means)
train_y = np.array(all_zones)
train_xs = np.array(all_means).T
return train_xs, train_y
def get_raster_value(centroid, out_image, out_transform):
"""Small function to obtain raster value from rastermap, using point_query.
Arguments:
*centroid* (shapely geometry) -- shapely geometry of centroid of the building.
*out_image* (numpy array) -- numpy array of grid.
*out_transform* (Affine) -- georeference of numpy array.
Returns:
*integer* -- raster value corresponding to location of centroid
"""
return int(
point_query(centroid,
out_image,
affine=out_transform,
nodata=-9999,
interpolate='nearest')[0] or 255)
def max_depth_from_raster(row, dem_pth, dem_adjustment=-4.63):
"""
return Max Depth for a df row passed with X & Y columns,
for use in a SWMM5 Junctions table
"""
try:
from rasterstats import point_query
except:
raise ImportError('rasterstats not found. Try pip install rasterstats')
if pd.isnull(row.X) or pd.isnull(row.Y):
return None
point = 'POINT({} {})'.format(row.X, row.Y)
rim_elev = point_query(point, dem_pth)[0] + dem_adjustment
invert = row.InvertElev
max_depth = rim_elev - invert
return max_depth
def integrate_shapes_sur(filename: str, shapefile: str) -> gpd.GeoDataFrame:
"""
This functions opens a geotiff with desired data, converts to a raster,
integrate the data into polygons and returns a GeoDataFrame object.
Parameters:
cuencas_shp: Path to shapefile
Returns:
cuencas_gdf_ppn (GeoDataFrame): a geodataframe with cuerncas and ppn
"""
cuencas_gdf: gpd.GeoDataFrame = gpd.read_file(shapefile, encoding='utf-8')
df_zs = pd.DataFrame(point_query(shapefile, filename))
cuencas_gdf_ppn = pd.concat([cuencas_gdf, df_zs], axis=1)
COLUM_REPLACE = {0: 'data'}
cuencas_gdf_ppn = cuencas_gdf_ppn.rename(columns=COLUM_REPLACE)
return cuencas_gdf_ppn[['NAME', 'geometry', 'data']]
def get_raster_value(centroid, raster_image, out_transform):
"""
Function to extract a value from a raster file for a given shapely Point.
Arguments:
*centroid* : shapely Point
*raster_image* : NumPy array, read by rasterio.
*out_transform* : Affine of **raster_image*.
Returns:
*int* : integer value from the raster at the location of the **centroid**. Will return **zero** when there is no intersection.
"""
return int(
point_query(centroid,
raster_image,
affine=out_transform,
nodata=-9999,
interpolate='nearest')[0] or 0)
def ExtractPointValues(raster, shp, ID):
from rasterstats import point_query
""" Extract raster values by a shapefile point mask.
"""
# Raster management
r = rasterio.open(raster)
affine = r.affine
array = r.read()
bands = array.shape[0]
bandNames = []
for i in range(bands):
a = "B" + str(i+1)
bandNames.append(a)
# Shapefile management
shape = shapefile.Reader(shp)
records = pd.DataFrame(shape.records())
n = pd.DataFrame(shape.fields)[0].values.tolist().index(ID)
id = records[n-1]
# empty matrix to store results
matrix = np.empty((len(records), bands+1), dtype=object)
matrix[:,0] = id
colnamesSHP = [ID]
# Final colnames
colNames = colnamesSHP + bandNames
# Extract values
for i in range(bands):
# stats
array = r.read(i+1)
stats = point_query(shp, array, affine=affine)
x = pd.DataFrame(stats)
matrix[:,i+1] = x[0]
# set the final data frame
df = pd.DataFrame(matrix, columns=colNames)
return df
def core(path, feature):
print('Computing {}'.format(path))
import glob
from rasterstats import point_query
from pathos import multiprocessing as mp
l = glob.glob(os.path.join(path, '*.tif'))
monyear = list(map(get_mon_year, l))
df = pd.DataFrame(monyear)
df = df.sort_values(by=['year', 'month'], ascending=[1, 1])
l = df.fn.tolist()
pool = mp.Pool(16)
out = pool.map(lambda x: point_query(feature.geometry, x), l)
pool.close()
pool.join()
df['date'] = df['year'].map(str) + '-' + df['month']
df = df.set_index(pd.to_datetime(df['date']))
df = df.drop(['year', 'month', 'fn', 'date'], 1)
name = path.split('/')[-3]
df[name] = [i for i in out]
return df
def ExtractPointValues(raster, shp, ID):
from rasterstats import point_query
""" Extract raster values by a shapefile point mask.
"""
# Raster management
r = rasterio.open(raster)
affine = r.affine
array = r.read()
bands = array.shape[0]
bandNames = []
for i in range(bands):
a = "B" + str(i+1)
bandNames.append(a)
# Shapefile management
shape = shapefile.Reader(shp)
records = pd.DataFrame(shape.records())
n = pd.DataFrame(shape.fields)[0].values.tolist().index(ID)
id = records[n-1]
# empty matrix to store results
matrix = np.empty((len(records), bands+1), dtype=object)
matrix[:,0] = id
colnamesSHP = [ID]
# Final colnames
colNames = colnamesSHP + bandNames
# Extract values
for i in range(bands):
# stats
array = r.read(i+1)
stats = point_query(shp, array, affine=affine)
x = pd.DataFrame(stats)
matrix[:,i+1] = x[0]
# set the final data frame
df = pd.DataFrame(matrix, columns=colNames)
return df
def pointquery(input_geojson, raster, output_geojson, band, indent, nodata,
interpolate, property_name):
"""
Queries the raster values at the points of the input GeoJSON Features.
The raster values are added to the features properties and output as GeoJSON
Feature Collection.
If the Features are Points, the point geometery is used.
For other Feauture types, all of the verticies of the geometry will be queried.
For example, you can provide a linestring and get the profile along the line
if the verticies are spaced properly.
You can use either bilinear (default) or nearest neighbor interpolation.
"""
mapping = json.loads(input_geojson.read())
input_geojson.close()
try:
if mapping['type'] == "FeatureCollection":
feature_collection = mapping
else:
feature_collection = {'type': 'FeatureCollection'}
features = read_features(mapping)
except (AssertionError, KeyError):
raise ValueError("input_geojson must be valid GeoJSON")
results = point_query(
features,
raster,
band=band,
nodata=nodata,
interpolate=interpolate,
property_name=property_name,
geojson_out=True)
feature_collection['features'] = results
output_geojson.write(json.dumps(feature_collection, indent=indent))
output_geojson.write("\n")
def gradient(route_shp, rasterfile):
"""
Calculates the elevation and road grade at each point along the route.
Parameters
----------
route_shp: shapefile for the selected route; output of read_shape()
rasterfile: elevation data file (.tif)
Returns
-------
elevation_meters: the elevation at each point along the route
route_gradient: the road grade (e.g. slope) at each point along the route
route_cum_distance: the total route distance at each point along the route [m]
route_distance: the distance between each point [m]
"""
elevation = rasterstats.point_query(route_shp, rasterfile)
elevation_meters = np.asarray(elevation) * 0.3048
route_distance, route_cum_distance = distance_measure(route_shp)
route_gradient = np.insert(abs(np.diff(elevation)/ route_distance),0, 0)
return elevation_meters, route_gradient, route_cum_distance, route_distance
def getTimeSeriesData(startingDate, endingDate, product, points):
## Get avalable dates
productLocation = getProductLocation(product)
filesList = os.listdir(productLocation)
## Check if datesList elementi is a valid directory
datesList = []
for anything in filesList:
if os.path.isdir(f'{productLocation}/{anything}'):
datesList.append(anything)
else:
pass
## Get list of dates between timeserise endig and starting dates
validDates = getDatesBetweenTwoDates(startingDate, endingDate, datesList)
## Returned object structure (customised for chart.js timeseries chart)
timeseries = {"datasets": []}
## Loop first the points list...
for point in points:
point = ast.literal_eval(point)
## ...and then valid dates
data = []
for dateString in validDates:
dateToReturn = f'{dateString[0:4]}-{dateString[4:6]}-{dateString[6:8]}'
## Construct the path to dateset
rasterFolderPath = f'{productLocation}/{dateString}/{product}/'
raster = os.listdir(rasterFolderPath)[0]
pt = Point(point["x"], point["y"])
if raster.endswith('tif'):
productValue = point_query([pt], f'{rasterFolderPath}/{raster}')
else:
productValue = None
data.append({"x": dateToReturn, "y": productValue})
data.sort(key = lambda x: datetime.strptime(x["x"], '%Y-%m-%d'))
timeseries["datasets"].append({"fid": point["id"], "label": f'{product}, x: {round(point["x"], 4)}, y: {round(point["y"], 4)}', "data": data})
return timeseries
with rasterio.open('./data/wc2-5/stack.tif', 'w', **meta) as dst:
for id, layer in enumerate(file_list, start=1):
with rasterio.open(layer) as src1:
dst.write_band(id, src1.read(1))
"""
print("\nNow reading a database of world climate...\n")
with rio.open('./data/wc2-5/stack.tif') as clim_src:
clim = clim_src.read(masked=True)
clim_meta = clim_src.profile
# extract raster values at points
bioclim_cols = []
for n in range(clim.shape[0]):
city_clim = rs.point_query(cities,
clim[n, :, :],
affine=clim_meta['transform'],
nodata=-9999)
bioclim_cols.append('bio%i' % (n + 1))
cities.loc[:, 'bio%i' % (n + 1)] = city_clim
# drop NANs
cities = cities.dropna()
# standardize the climate columns
bioclim_data = cities.loc[:, bioclim_cols].values
bioclim_data = StandardScaler().fit_transform(bioclim_data)
# run climate PCA, add as columns to cities
print(("\nNow doing some data science (running PCA, "
"then finding your city's nearest neighbors)...\n"))
pca = PCA(n_components=19)
def zstats_partial(feats):
"""
Imports raster values into a dataframe partition and returns the partition
Arguments:
feats (array)-> partion of dataframe
Output: returns a gridded dataframe
"""
# Get all tif rasters in folder 'rasters'
folder = 'rasters/'
rasters = glob(folder + '*.tif')
feats['centroid_column'] = feats.centroid
# Loop over rasters to create a dictionary of raster path and raster name for column names and add values to grid
for i in rasters:
# Get data type of raster
with rasterio.open(i) as dataset:
col_dtype = dataset.meta['dtype']
# Extract column name from raster name
colname = os.path.basename(i).split('.')[0]
logger.info(" Col name {} dtype {}".format(colname, col_dtype))
# Query effective temperature
if 'efftemp' in colname:
stats = point_query(feats.set_geometry('centroid_column'),
i,
interpolate='nearest')
feats[colname] = pd.Series([d for d in stats],
index=feats.index,
dtype=col_dtype)
# Query landcover and apply lc_summary function
elif colname == 'landcover':
# For land cover raster, count the number of covers in each cell
stats = zonal_stats(feats.set_geometry('geometry'),
i,
categorical=True)
result = list(map(lc_summary, stats, feats['area'].values))
for cols, pos in zip([
'est_area', 'suitable_area', "pasture_area", "crop_area",
'tree_area'
], range(5)):
feats[cols] = [i[pos] for i in result]
# Get mean accessibility
elif colname == 'accessibility':
stats = zonal_stats(feats.set_geometry('geometry'),
i,
stats='mean',
nodata=-9999)
feats[colname] = pd.Series([d['mean'] for d in stats],
index=feats.index)
#For all other rasters do a point query instead of zonal statistics and replace negative values by NaN
else:
stats = point_query(feats.set_geometry('centroid_column'),
i,
interpolate='nearest')
feats[colname] = pd.Series(
[0 if d is None else 0 if d < 0 else d for d in stats],
index=feats.index,
dtype=col_dtype)
logger.info(" Done with " + colname)
# Establishment cost of 8 ('000)$ per ha where land cover requires a transition (not grass or crop) from
# (Dietrich et al 2019 Geosci. Model Dev.
feats['est_cost'] = feats['est_area'] * 8
feats["opp_cost"] = feats["agri_opp_cost"] + feats['ls_opp_cost'] * 0.01
logger.info("Done with opp_cost")
feats["nutrient_availability"] = feats['nutrient_availability'].replace(
0, 2)
return feats
# -*- coding: utf-8 -*-
# skip this test because one band proccesing takes more than 6 min
if False:
import os
from rasterstats import point_query
wd = os.getcwd()
catalog = os.path.join('data', 'LC08_L1TP_190024_20200418_20200822_02_T1')
rasters = os.listdir(catalog)
rasters = [r for r in rasters if r.endswith(('.TIF'))]
rasters = [os.path.join(wd, catalog, r) for r in rasters]
band_names = ["B1", "B10", "B11", "B2", "B3", "B4", "B5", "B6", "B7", "B9"]
points = os.path.join('data', 'vector', 'points.gpkg')
### extract
data = []
for ras in rasters:
vals = point_query(points, ras, interpolate='nearest')
data.append(vals)
def PhenoPlot(X,
Y,
inData,
dates,
type='KDE',
saveFigure=None,
ylim=None,
rollWindow=None,
nan_replace=None,
correctionValue=None,
plotType=1,
phentype=1,
nGS=46,
n_phen=15,
fontsize=14,
titlesize=15,
legendsize=15,
labelsize=13,
threshold=300,
ylab='NDVI'):
"""
Plot the PhenoShape curve along with the yearly data
Parameters
----------
- X: Float
X coordinates
- Y: Float
Y coordinates
- inData: String
Absolute path to the original timeseries data
- dates: Series
Dates of the original timeseries data [dtype: datetime64[ns]]
- type = String or Integer
Interpolation type. Must be a string of ‘linear’, ‘nearest’,
‘zero’, ‘slinear’, ‘quadratic’, ‘cubic’, ‘RBF‘, ‘previous’,
‘next’, where ‘zero’, ‘slinear’, ‘quadratic’ and ‘cubic’ refer
to a spline interpolation of zeroth, first, second or third order;
‘previous’ and ‘next’ simply return the previous or next value"
of the point) or as an integer specifying the order of the"
spline interpolator to use. RBF uses cubic interpolation.
Default is ‘linear’.
- saveFigure: String
Absolute path with extention to save figure on disk
- ylim: List of Integers or Float
Limits of the Y axis [default the y min() and max() values]
- plotType: Type of plot, where 1 = plot with accumulated years; 2 = plot with
start of the season (SOS), peak of the season (POS) and end of
season (EOS); and 3 = KDE plot
default is 1
- rollWindow: Integers
Value of avarage smoothing of linear trend [default None]
- nGS: Integer
Number of observations to predict the PhenoShape
default is 46; one per week
- ylab: string
Label of the Y axis [default "NDVI"]
"""
# get spatial point
point = geom.Point(X, Y)
# get pixel value per pixel
# first read metadata to get number of bands
with rasterio.open(inData) as r:
countTSS = r.count
# save values
valuesTSS = []
for i in range(countTSS):
valuesTSS.append(point_query(point, inData, band=(i + 1)))
valuesTSS = np.array(valuesTSS, dtype=np.float).squeeze()
if nan_replace is not None:
valuesTSS = np.where(valuesTSS == nan_replace, np.nan, valuesTSS)
if correctionValue is not None:
valuesTSS = valuesTSS / correctionValue
# add dates
valuesTSSpd = pd.concat([
pd.DataFrame(dates),
pd.DataFrame(dates.dt.dayofyear),
pd.DataFrame(dates.dt.year),
pd.DataFrame(valuesTSS)
],
axis=1)
valuesTSSpd.columns = ['dates', 'doy', 'year', 'VI']
valuesTSSpd = valuesTSSpd.sort_values('doy')
# group values according to year
groups = valuesTSSpd.groupby('year')
# get phenological shape
phen = _getPheno0(y=valuesTSS,
doy=dates.dt.dayofyear,
interpolType=type,
nan_replace=None,
rollWindow=rollWindow,
nGS=nGS)
# doy of the predicted phenological shape
xnew = np.linspace(np.min(valuesTSSpd.doy),
np.max(valuesTSSpd.doy),
nGS,
dtype='int16')
# plot
if plotType == 1:
# get %RMSE
rmse = _RMSE(valuesTSSpd.doy.values, valuesTSSpd.VI.values, xnew,
phen) #.round(2)
minn = np.nanmin(valuesTSS)
maxx = np.nanmax(valuesTSS)
nRMSE = ((rmse / (maxx - minn)) * 100).round(2)
for name, group in groups:
plt.plot(group.doy,
group.VI,
marker='o',
linestyle='',
ms=10,
label=name)
plt.plot(xnew, phen, '-', color='black')
plt.legend(prop={'size': legendsize})
plt.tick_params(labelsize=labelsize)
plt.title('%RMSE = ' + str(nRMSE), loc='left', size=titlesize)
if ylim is not None:
plt.ylim(ylim[0], ylim[1])
plt.ylabel(ylab, fontsize=fontsize)
plt.xlabel('Day of the year', fontsize=fontsize)
elif plotType == 2:
# get position of SOS, POS, and EOS
metrics = _getLSPmetrics(phen, xnew, nGS, len(xnew), phentype)
isos = np.where(xnew == metrics[0])[0][0]
ipos = np.where(xnew == metrics[1])[0][0]
ieos = np.where(xnew == metrics[2])[0][0]
plt.plot(xnew, phen, '-', color='black')
plt.plot(xnew[isos], phen[isos], 'X', markersize=15, label='SOS')
plt.plot(xnew[ipos], phen[ipos], 'X', markersize=15, label='POS')
plt.plot(xnew[ieos], phen[ieos], 'X', markersize=15, label='EOS')
plt.legend(prop={'size': 12})
if ylim is not None:
plt.ylim(ylim[0], ylim[1])
plt.ylabel(ylab, fontsize=fontsize)
plt.xlabel('Day of the year', fontsize=fontsize)
if saveFigure is not None:
plt.savefig(saveFigure)
plt.show()
def test_point_query_geojson():
point = "POINT(245309 1000064)"
features = point_query(point, raster, property_name="TEST", geojson_out=True)
for feature in features:
assert 'TEST' in feature['properties']
assert round(feature['properties']['TEST']) == 74
def test_point_query():
point = "POINT(245309 1000064)"
val = point_query(point, raster)[0]
assert round(val) == 74
