def convert_tif_to_wgs84(input, output):
'''convert from input to output to wgs84 CRS'''
print(f'convert| {input}\n\t->{output}')
dst_crs = 'EPSG:4326'
with rio_open(input) as src:
transform, width, height = calculate_default_transform(
src.crs, dst_crs, src.width, src.height, *src.bounds)
kwargs = src.meta.copy()
kwargs.update({
'crs': dst_crs,
'transform': transform,
'width': width,
'height': height
})
with rio_open(output, 'w', **kwargs) as dst:
for i in range(1, src.count + 1):
reproject(
source=band(src, i),
destination=band(dst, i),
src_transform=src.transform,
src_crs=src.crs,
dst_transform=transform,
dst_crs=dst_crs,
resampling=Resampling.nearest)
def mosaic(images_paths):
output = images_paths[0].replace('part0-', '')
files_to_mosaic = [rio_open(path) for path in images_paths]
mosaic, out_trans = merge(files_to_mosaic)
out_profile = files_to_mosaic[0].profile.copy()
out_profile.update({"height": mosaic.shape[1], "width": mosaic.shape[2], "transform": out_trans})
with rio_open(output, 'w', **out_profile) as dataset:
dataset.write(mosaic)
for path, file in zip(images_paths, files_to_mosaic):
file.close()
os.remove(path)
def mosaic(images_paths, delete=True):
output = images_paths[0].replace('part0-', '')
files_to_mosaic = [rio_open(path) for path in images_paths]
mosaic, out_trans = merge(files_to_mosaic)
out_profile = files_to_mosaic[0].profile.copy()
out_profile.update({
"height": mosaic.shape[1],
"width": mosaic.shape[2],
"transform": out_trans
})
print('preparing to write mosaic')
with rio_open(output, 'w', **out_profile) as dataset:
dataset.write(mosaic)
print(f'mosaic created in: {output}')
if delete:
for path, file in zip(images_paths, files_to_mosaic):
file.close()
os.remove(path)
print('Temporary files deleted')
def outputFiles(crs, masks):
"""
* Output illustrative GIS (Shapefile and GeoTiff) files of the extracted
* zones and the aoi polygons
"""
# create the file in write mode, with the required CRS (Proj4 string) and an empty schema
with fi_open('./tmp/aoi.shp',
'w',
driver='ESRI Shapefile',
crs=crs,
schema={
'geometry': 'Polygon',
'properties': {}
}) as o:
# loop through resulting masks
for m in range(len(masks)):
# write the splitter to shapefile
o.write({'geometry': mapping(masks[m]["aoi"]), 'properties': {}})
# create a raster file
with rio_open(
f'./tmp/{m}.tif',
'w',
driver=masks[m]["meta"]["driver"],
height=masks[m]["meta"]["height"],
width=masks[m]["meta"]["width"],
count=masks[m]["meta"]["count"],
dtype=masks[m]["meta"]["dtype"],
crs=masks[m]["meta"]["crs"],
transform=masks[m]["meta"]["transform"],
) as dst:
# write the data to the raster
dst.write(masks[m]["dtm"], 1)
def concat(filenames,
stack_dim='time',
bounds_by='reference',
resampling='nearest',
time_names=None,
band_names=None,
nodata=None,
dtype=None,
overlap='max',
warp_mem_limit=512,
num_threads=1,
tap=False,
**kwargs):
"""
Concatenates a list of images
Args:
filenames (list): A list of file names to concatenate.
stack_dim (Optional[str]): The stack dimension. Choices are ['time', 'band'].
bounds_by (Optional[str]): How to concatenate the output extent. Choices are ['intersection', 'union', 'reference'].
* reference: Use the bounds of the reference image
* intersection: Use the intersection (i.e., minimum extent) of all the image bounds
* union: Use the union (i.e., maximum extent) of all the image bounds
resampling (Optional[str]): The resampling method.
time_names (Optional[1d array-like]): A list of names to give the time dimension.
band_names (Optional[1d array-like]): A list of names to give the band dimension.
nodata (Optional[float | int]): A 'no data' value to set. Default is None.
dtype (Optional[str]): A data type to force the output to. If not given, the data type is extracted
from the file.
overlap (Optional[str]): The keyword that determines how to handle overlapping data.
Choices are ['min', 'max', 'mean']. Only used when mosaicking arrays from the same timeframe.
warp_mem_limit (Optional[int]): The memory limit (in MB) for the ``rasterio.vrt.WarpedVRT`` function.
num_threads (Optional[int]): The number of warp worker threads.
tap (Optional[bool]): Whether to target align pixels.
kwargs (Optional[dict]): Keyword arguments passed to ``xarray.open_rasterio``.
Returns:
``xarray.DataArray``
"""
if stack_dim.lower() not in ['band', 'time']:
logger.exception(" The stack dimension should be 'band' or 'time'")
ref_kwargs = {
'bounds': None,
'crs': None,
'res': None,
'nodata': nodata,
'warp_mem_limit': warp_mem_limit,
'num_threads': num_threads,
'tap': tap,
'tac': None
}
ref_kwargs = _check_config_globals(filenames, bounds_by, ref_kwargs)
with rio_open(filenames[0]) as src_:
tags = src_.tags()
# Keep a copy of the transformed attributes.
with xr.open_rasterio(
warp(filenames[0], resampling=resampling, **ref_kwargs),
**kwargs) as src_:
attrs = src_.attrs.copy()
if time_names:
concat_list = list()
new_time_names = list()
# Check the time names for duplicates
for tidx in range(0, len(time_names)):
if list(time_names).count(time_names[tidx]) > 1:
if time_names[tidx] not in new_time_names:
# Get the file names to mosaic
filenames_mosaic = [
filenames[i] for i in range(0, len(time_names))
if time_names[i] == time_names[tidx]
]
# Mosaic the images into a single-date array
concat_list.append(
mosaic(filenames_mosaic,
overlap=overlap,
bounds_by=bounds_by,
resampling=resampling,
band_names=band_names,
nodata=nodata,
warp_mem_limit=warp_mem_limit,
num_threads=num_threads,
**kwargs))
new_time_names.append(time_names[tidx])
else:
new_time_names.append(time_names[tidx])
# Warp the date
concat_list.append(
warp_open(filenames[tidx],
resampling=resampling,
band_names=band_names,
nodata=nodata,
warp_mem_limit=warp_mem_limit,
num_threads=num_threads,
**kwargs))
# Warp all images and concatenate along the 'time' axis into a DataArray
output = xr.concat(concat_list, dim=stack_dim.lower())
# Assign the new time band names
src = output.assign_coords(time=new_time_names)
else:
# Warp all images and concatenate along
# the 'time' axis into a DataArray.
src = xr.concat([
xr.open_rasterio(warp(fn, resampling=resampling, **ref_kwargs), **
kwargs) for fn in filenames
],
dim=stack_dim.lower())
src.attrs['filename'] = [Path(fn).name for fn in filenames]
if tags:
attrs = src.attrs.copy()
attrs.update(tags)
src = src.assign_attrs(**attrs)
if not time_names and (stack_dim == 'time'):
src.coords['time'] = parse_filename_dates(filenames)
if band_names:
src.coords['band'] = band_names
else:
if src.gw.sensor:
if src.gw.sensor not in src.gw.avail_sensors:
logger.warning(
' The {} sensor is not currently supported.\nChoose from [{}].'
.format(src.gw.sensor, ', '.join(src.gw.avail_sensors)))
else:
new_band_names = list(
src.gw.wavelengths[src.gw.sensor]._fields)
if len(new_band_names) != len(src.band.values.tolist()):
if not src.gw.config['ignore_warnings']:
logger.warning(
' The new bands, {}, do not match the sensor bands, {}.'
.format(new_band_names, src.band.values.tolist()))
else:
src.coords['band'] = new_band_names
src.attrs['sensor'] = src.gw.sensor_names[src.gw.sensor]
if dtype:
attrs = src.attrs.copy()
return src.astype(dtype).assign_attrs(**attrs)
else:
return src
def mosaic(filenames,
overlap='max',
bounds_by='reference',
resampling='nearest',
band_names=None,
nodata=None,
dtype=None,
warp_mem_limit=512,
num_threads=1,
**kwargs):
"""
Mosaics a list of images
Args:
filenames (list): A list of file names to mosaic.
overlap (Optional[str]): The keyword that determines how to handle overlapping data.
Choices are ['min', 'max', 'mean'].
bounds_by (Optional[str]): How to concatenate the output extent. Choices are ['intersection', 'union', 'reference'].
* reference: Use the bounds of the reference image
* intersection: Use the intersection (i.e., minimum extent) of all the image bounds
* union: Use the union (i.e., maximum extent) of all the image bounds
resampling (Optional[str]): The resampling method.
band_names (Optional[1d array-like]): A list of names to give the band dimension.
nodata (Optional[float | int]): A 'no data' value to set. Default is None.
dtype (Optional[str]): A data type to force the output to. If not given, the data type is extracted
from the file.
warp_mem_limit (Optional[int]): The memory limit (in MB) for the ``rasterio.vrt.WarpedVRT`` function.
num_threads (Optional[int]): The number of warp worker threads.
kwargs (Optional[dict]): Keyword arguments passed to ``xarray.open_rasterio``.
Returns:
``xarray.DataArray``
"""
if overlap not in ['min', 'max', 'mean']:
logger.exception(
" The overlap argument must be one of ['min', 'max', 'mean'].")
ref_kwargs = {
'bounds': None,
'crs': None,
'res': None,
'nodata': nodata,
'warp_mem_limit': warp_mem_limit,
'num_threads': num_threads,
'tac': None
}
ref_kwargs = _check_config_globals(filenames, bounds_by, ref_kwargs)
# Warp all images to the same grid.
warped_objects = warp_images(filenames,
bounds_by=bounds_by,
resampling=resampling,
**ref_kwargs)
footprints = []
with rio_open(filenames[0]) as src_:
tags = src_.tags()
# Combine the data
with xr.open_rasterio(warped_objects[0], **kwargs) as darray:
attrs = darray.attrs.copy()
# Get the original bounds, unsampled
with xr.open_rasterio(filenames[0], **kwargs) as src_:
footprints.append(src_.gw.geometry)
for fidx, fn in enumerate(warped_objects[1:]):
with xr.open_rasterio(fn, **kwargs) as darrayb:
with xr.open_rasterio(filenames[fidx + 1], **kwargs) as src_:
footprints.append(src_.gw.geometry)
src_ = None
if overlap == 'min':
if isinstance(nodata, float) or isinstance(nodata, int):
darray = xr.where(
(darray.mean(dim='band') == nodata) &
(darrayb.mean(dim='band') != nodata), darrayb,
xr.where((darray.mean(dim='band') != nodata) &
(darrayb.mean(dim='band') == nodata),
darray, xr_mininum(darray, darrayb)))
else:
darray = xr_mininum(darray, darrayb)
elif overlap == 'max':
if isinstance(nodata, float) or isinstance(nodata, int):
darray = xr.where(
(darray.mean(dim='band') == nodata) &
(darrayb.mean(dim='band') != nodata), darrayb,
xr.where((darray.mean(dim='band') != nodata) &
(darrayb.mean(dim='band') == nodata),
darray, xr_maximum(darray, darrayb)))
else:
darray = xr_maximum(darray, darrayb)
elif overlap == 'mean':
if isinstance(nodata, float) or isinstance(nodata, int):
darray = xr.where(
(darray.mean(dim='band') == nodata) &
(darrayb.mean(dim='band') != nodata), darrayb,
xr.where((darray.mean(dim='band') != nodata) &
(darrayb.mean(dim='band') == nodata),
darray, (darray + darrayb) / 2.0))
else:
darray = (darray + darrayb) / 2.0
# darray = darray.combine_first(darrayb)
darray = darray.assign_attrs(**attrs)
if band_names:
darray.coords['band'] = band_names
else:
if darray.gw.sensor:
if darray.gw.sensor not in darray.gw.avail_sensors:
logger.warning(
' The {} sensor is not currently supported.\nChoose from [{}].'
.format(darray.gw.sensor,
', '.join(darray.gw.avail_sensors)))
else:
new_band_names = list(
darray.gw.wavelengths[darray.gw.sensor]._fields)
if len(new_band_names) != len(darray.band.values.tolist()):
logger.warning(
' The band list length does not match the sensor bands.'
)
else:
darray.coords['band'] = new_band_names
darray.attrs['sensor'] = darray.gw.sensor_names[
darray.gw.sensor]
darray.attrs['filename'] = [Path(fn).name for fn in filenames]
darray.attrs['resampling'] = resampling
if tags:
attrs = darray.attrs.copy()
attrs.update(tags)
darray = darray.assign_attrs(**attrs)
darray.gw.footprint_grid = footprints
if dtype:
attrs = darray.attrs.copy()
return darray.astype(dtype).assign_attrs(**attrs)
else:
return darray
def warp_open(filename,
band_names=None,
nodata=None,
resampling='nearest',
dtype=None,
return_windows=False,
warp_mem_limit=512,
num_threads=1,
tap=False,
**kwargs):
"""
Warps and opens a file
Args:
filename (str): The file to open.
band_names (Optional[int, str, or list]): The band names.
nodata (Optional[float | int]): A 'no data' value to set. Default is None.
resampling (Optional[str]): The resampling method.
dtype (Optional[str]): A data type to force the output to. If not given, the data type is extracted
from the file.
return_windows (Optional[bool]): Whether to return block windows.
warp_mem_limit (Optional[int]): The memory limit (in MB) for the ``rasterio.vrt.WarpedVRT`` function.
num_threads (Optional[int]): The number of warp worker threads.
tap (Optional[bool]): Whether to target align pixels.
kwargs (Optional[dict]): Keyword arguments passed to ``xarray.open_rasterio``.
Returns:
``xarray.DataArray``
"""
ref_kwargs = {
'bounds': None,
'crs': None,
'res': None,
'nodata': nodata,
'warp_mem_limit': warp_mem_limit,
'num_threads': num_threads,
'tap': tap,
'tac': None
}
ref_kwargs = _check_config_globals(filename, 'reference', ref_kwargs)
with rio_open(filename) as src:
tags = src.tags()
with xr.open_rasterio(warp(filename, resampling=resampling, **ref_kwargs),
**kwargs) as src:
if band_names:
src.coords['band'] = band_names
else:
if src.gw.sensor:
if src.gw.sensor not in src.gw.avail_sensors:
logger.warning(
' The {} sensor is not currently supported.\nChoose from [{}].'
.format(src.gw.sensor,
', '.join(src.gw.avail_sensors)))
else:
new_band_names = list(
src.gw.wavelengths[src.gw.sensor]._fields)
# Avoid nested opens within a `config` context
if len(new_band_names) != len(src.band.values.tolist()):
if not src.gw.config['ignore_warnings']:
logger.warning(
' The new bands, {}, do not match the sensor bands, {}.'
.format(new_band_names,
src.band.values.tolist()))
else:
src.coords['band'] = new_band_names
src.attrs['sensor'] = src.gw.sensor_names[
src.gw.sensor]
if return_windows:
if isinstance(kwargs['chunks'], tuple):
chunksize = kwargs['chunks'][-1]
else:
chunksize = kwargs['chunks']
src.attrs['block_windows'] = get_window_offsets(src.shape[-2],
src.shape[-1],
chunksize,
chunksize,
return_as='list')
src.attrs['filename'] = filename
src.attrs['resampling'] = resampling
if tags:
attrs = src.attrs.copy()
attrs.update(tags)
src = src.assign_attrs(**attrs)
if dtype:
attrs = src.attrs.copy()
return src.astype(dtype).assign_attrs(**attrs)
else:
return src
def f(mask, aoi_id, plot=False):
'''
* run parallel process
'''
# turn off pymc3 logging
getLogger("pymc3").setLevel(ERROR)
# get transform object for the dataset (nw corner & resolution)
transform = from_origin(mask['bounds'][0], mask['bounds'][3],
mask['resolution'], mask['resolution'])
# check that output directory is there
if not exists("./out/"):
makedirs("./out/")
# seed data and uncertainty arrays for the study area and build dictionary to control outputs
c_data = zeros(
(ceil((mask['bounds'][3] - mask['bounds'][1]) / mask['resolution']),
ceil((mask['bounds'][2] - mask['bounds'][0]) / mask['resolution'])))
outputs = {
'catholic': {
'path': f'./out/{aoi_id}_catholic.tif',
'mean': c_data,
'low': c_data.copy(),
'high': c_data.copy()
},
'protestant': {
'path': f'./out/{aoi_id}_protestant.tif',
'mean': c_data.copy(),
'low': c_data.copy(),
'high': c_data.copy()
},
'mixed': {
'path': f'./out/{aoi_id}_mixed.tif',
'mean': c_data.copy(),
'low': c_data.copy(),
'high': c_data.copy()
}
}
# extract list of group names
groups = array(list(outputs.keys()))
# use try-finally so if it fails we can see where it got up to
# try:
print(f"AOI Dimensions: {c_data.shape[1]}x{c_data.shape[0]}px")
# loop through rows and columns in the dataset
for row in range(c_data.shape[0]):
for col in range(c_data.shape[1]):
print(
f"\t...{row * c_data.shape[1] + col} of {c_data.shape[0] * c_data.shape[1]} ({(row * c_data.shape[1] + col)/(c_data.shape[0] * c_data.shape[1])*100:.2f}%)"
)
# get coordinates for the point
point = Point(array2Coords(transform, row, col))
''' calculate hyperparameters (priors) '''
# get the census data for the census Small Area that contains the point
possible_matches = mask['census'].iloc[list(
mask['census'].sindex.intersection(point.bounds))]
district = possible_matches.loc[possible_matches.contains(point)][[
'pcCatholic', 'pcProtesta', 'pc_Other', 'pc_None'
]]
# make sure that there was a match at all!
if len(district.index) > 0:
# compute proportions for the three groups
# replace zeros for 1s as you are not allowed 0's in the hyperparameters (gives Bad initial energy error)
alphas = maximum(
ones(3),
array([
int(round(district['pcCatholic'].iloc[0])),
int(round(district['pcProtesta'].iloc[0])),
int(
round(district['pc_Other'].iloc[0] +
district['pc_None'].iloc[0]))
]))
else:
# if no matches, have equal belief for each group
alphas = array([1, 1, 1])
''' calculate observations '''
# init lists for observations
c = []
n = []
# construct the radius for analysis
polygon = point.buffer(mask['radius'])
# loop through each dataset
for i, gdf in mask['datasets'].items():
# check that there is data available (this is if no data has been
# passed in the mask as the clip polygon does not intersect any)
if len(gdf.index) > 0:
# get data points within and get IDW2 multiplier
possible_matches = gdf.iloc[list(
gdf.sindex.intersection(polygon.bounds))]
observations = possible_matches.loc[
possible_matches.within(polygon)]
observations['idw2'] = (
1 - observations.geometry.distance(point) /
mask['radius'])**2
# check that there is data available (this is if data has been
# passed but the buffer polygon does not intersect it)
if len(observations) > 0:
# get weighted group counts for the current dataset
if i == 'mapme':
catholics, protestants, mixed = getMapmeGroups(
observations)
elif i == 'gps':
catholics, protestants, mixed = getGpsGroups(
observations)
elif i == 'survey':
catholics, protestants, mixed = getSurveyGroups(
observations)
# index and int the scores for each dataset
sums = [catholics, protestants, mixed]
# catch error caused by no probabilities
if sum(sums) > 0:
print(sums)
# process into correct format
sums = [
int(round(i / sum(sums) * 100)) for i in sums
]
# append to observations list
c.append(sums)
n.append(sum(sums))
# TODO: DO I WANT ALL THESE 0'S OR ARE THEY GOING TO CAUSE PROBLEMS?
else:
# if not matches, just append some empty data
c.append([0, 0, 0])
n.append(0)
else:
# if not matches, just append some empty data
c.append([0, 0, 0])
n.append(0)
else:
# if not matches, just append some empty data
c.append([0, 0, 0])
n.append(0)
# convert observations np array
c = array(c)
n = array(n)
# print(alphas, c, n)
# print()
''' run model '''
# start making MCC model
with Model() as model:
# TODO: LOOK INTO TESTVALS FOR PARAMETERS
# https://nbviewer.jupyter.org/github/CamDavidsonPilon/Probabilistic-Programming-and-Bayesian-Methods-for-Hackers/blob/master/Chapter3_MCMC/Ch3_IntroMCMC_PyMC3.ipynb#Intelligent-starting-values
# parameters of the Multinomial are from a Dirichlet
parameters = Dirichlet('parameters', a=alphas, shape=3)
# observed data is from a Multinomial distribution
observed_data = Multinomial('observed_data',
n=n,
p=parameters,
shape=3,
observed=c)
with model:
# estimate the Maximum a Posterior
# start = find_MAP() #don't use this - it prevents convergence!
# sample from the posterior (NUTS is default so is not explicitly stated)
trace = sample(
# start=start, # start at the MAP to increase chance of convergence -- DON'T DO THIS!
draws=1000, # number of sample draws
chains=
4, # number of chains in which the above are drawn (match cores)
cores=1, # max permitted by library
tune=500, # how many will be discarded (>=50% of draws)
discard_tuned_samples=True, # discard the tuning samples
progressbar=
False, # avoid unnecessarilly filling up the output file
target_accept=
0.9 # up from 0.8 to avoid false positives: https://eigenfoo.xyz/bayesian-modelling-cookbook/#fixing-divergences
)
if plot:
plot_trace(trace, show=True)
# retrieve summary data
results = summary(trace)
results.index = groups
# output the result to the datasets
for k, v in outputs.items():
v['mean'][row, col] = results.loc[k, 'mean']
v['low'][row, col] = results.loc[k, 'hpd_3%']
v['high'][row, col] = results.loc[k, 'hpd_97%']
# if we get an error - print some debugging info
# except Exception as e:
# print("\n--- EXCEPTION ---")
# print(e)
# print(row, col, point)
# if (sums):
# print(sums)
# else:
# print("sums not defined yet")
# print(c, n)
#
# # whatever happens, output the results to files
# finally:
# loop through outputs
for g in outputs.values():
# output dataset to raster (hardcoded crs as was causing error)
with rio_open(g['path'],
'w',
driver='GTiff',
height=g['mean'].shape[0],
width=g['mean'].shape[1],
count=3,
dtype='float64',
crs="EPSG:29902",
transform=transform) as out:
# add data and uncertainties as raster bands
out.write(g['mean'], 1)
out.write(g['low'], 2)
out.write(g['high'], 3)
from glob import glob
from rasterio import open as rio_open
from rasterio.merge import merge as rio_merge
# get all tif images in directory
images = glob('../CEF-Result/out/*.tif')
# get files
files = []
for file in images:
files.append(rio_open(file, 'r'))
# merge result files
merged, out_transform = rio_merge(files, method='max')
# output dataset to raster (hardcoded crs as was causing error)
with rio_open("../CEF-Result/800m.tif", 'w', driver='GTiff', height=merged.shape[1],
width=merged.shape[2], count=1, dtype='float64', crs="EPSG:27700",
transform=out_transform) as out:
out.write(merged[0], 1)
# close all of the files
for file in files:
file.close()
print("done")
'set the name of the variable that contains the output JS object (default: \'data\')'
)
args = parser.parse_args()
# validate output file is JavaScript
if args.output[0][-3:] != ".js":
print(
"Sorry, " + args.output[0] +
" is not a JavaScript file - it should be named in the form: \"*.js\"")
exit()
# ready to catch errors
try:
# open input data set
with rio_open(args.input[0]) as ds:
# open output file
file = open(args.output[0], 'w')
# get list of bands if specified, otherwise set to all bands
if (args.bands):
bands = [int(x) for x in args.bands]
else:
bands = list(range(1, ds.count + 1))
# get projection string (from online if it is EPSG as proj4js doesn't like them)
if ds.crs.is_epsg_code:
projString = get("https://epsg.io/" + ds.crs.to_proj4()[11:] +
".proj4").text
else:
# log start time and log start
time = datetime.now()
print(
f"a {sections[0]}x{sections[1]} grid, {sections[0]*sections[1]} processes, started at {time}."
)
# initialise masks array for the results
masks = []
# get aoi bounds from shapefile
with fi_open(boundary_path) as boundary:
bounds = boundary.bounds
# read in raster data
with rio_open(dtm_path) as dtm_input:
with rio_open(dsm_path) as dsm_input:
with rio_open(green_path) as green_input:
# verify raster dimensions and resolution
if (dsm_input.width == dtm_input.width == green_input.width) == False or \
(dsm_input.height == dtm_input.height == green_input.height) == False or \
(dsm_input.res[0] == dtm_input.res[0] == green_input.res[0]) == False:
print("rasters do not match!")
print("width: \t\t",
dsm_input.width == dtm_input.width == green_input.width)
print(
"height: \t",
dsm_input.height == dtm_input.height == green_input.height)
print(
"resolution: \t",
def array(self):
with rio_open(self.path, 'r') as dataset:
return dataset.read(1)
def plot(self, **kwargs):
with rio_open(self.path, 'r') as dataset:
imshow(dataset.read(1), **kwargs)
def f(mask):
"""
* main function for running with parallel.py
"""
# create an output array at the same dimensions as data for output
gvi = zeros((mask["meta"]["height"], mask["meta"]["width"]))
# radius in pixels
radius_px = int(mask["options"]["radius"] // mask['meta']['transform'][0])
# build weighting mask
weighting_mask = zeros((radius_px*2, radius_px*2))
for r, c in column_stack(disk((radius_px, radius_px), radius_px, shape=weighting_mask.shape)):
weighting_mask[(r, c)] = exp(-0.0003 * (hypot(radius_px - c, radius_px - r) * mask['meta']['transform'][0]))
# get pixel references for aoi extents
min_r, min_c = coords2Array(mask["meta"]["transform"], mask["aoi"].bounds[0], mask["aoi"].bounds[3])
max_r, max_c = coords2Array(mask["meta"]["transform"], mask["aoi"].bounds[2], mask["aoi"].bounds[1])
# loop through dataset rows and columns
for r in range(min_r, max_r+1):
for c in range(min_c, max_c+1):
# print(r, c)
# t1_start = perf_counter()
# call (weighted) viewshed
output = viewshed(r, c, radius_px, # coords and radius in pixels
mask['meta']['transform'][0], # resolution of datasets
mask["options"]["o_height"], # observer height
mask["options"]["t_height"], # target height
mask["dsm"], # dsm dataset
mask["dtm"], # dtm dataset
mask["meta"]["transform"]) # affine transform
# extract the viewshed data from the output surface and apply weighting mask
visible = output[r-radius_px:r+radius_px, c-radius_px:c+radius_px] * weighting_mask
# print(f"\tviewshed took {perf_counter() - t1_start}s", visible.shape, visible.sum())
# t1_start = perf_counter()
# multiply extract of (weighted) viewshed with extract of (weighted) green dataset
visible_green = visible * (mask["green"][r-radius_px:r+radius_px, c-radius_px:c+radius_px] * weighting_mask)
# print(f"\tvisible green {perf_counter() - t1_start}s", visible_green.shape, visible_green.sum())
# t1_start = perf_counter()
# get the ratio for greenness in the view
gvi[(r,c)] = visible_green.sum() / visible.sum()
# print(f"\tgvi took {perf_counter() - t1_start}s", gvi[(r,c)])
# print()
# clip gvi to aoi bounds
gvi = gvi[min_r:max_r+1, min_c:max_c+1]
# check that tmp folder exists
if not exists('./tmp/'):
makedirs('tmp')
# make unique filename
filepath = f'./tmp/{str(uuid1())[:8]}.tif'
# output file with updated dimensions and transform
with rio_open(filepath, 'w',
driver = mask["meta"]["driver"],
height = gvi.shape[0],
width = gvi.shape[1],
count = mask["meta"]["count"],
dtype = 'float64',
crs = mask["meta"]["crs"],
transform = Affine(
mask['meta']['transform'][0],
mask['meta']['transform'][1],
mask["aoi"].bounds[0],
mask['meta']['transform'][3],
mask['meta']['transform'][4],
mask["aoi"].bounds[3]),
) as dst:
dst.write(gvi, 1)
# return the filepath to the result
return filepath
def warp_open(filename,
band_names=None,
resampling='nearest',
dtype=None,
netcdf_vars=None,
nodata=0,
return_windows=False,
warp_mem_limit=512,
num_threads=1,
tap=False,
**kwargs):
"""
Warps and opens a file
Args:
filename (str): The file to open.
band_names (Optional[int, str, or list]): The band names.
resampling (Optional[str]): The resampling method.
dtype (Optional[str]): A data type to force the output to. If not given, the data type is extracted
from the file.
netcdf_vars (Optional[list]): NetCDF variables to open as a band stack.
nodata (Optional[float | int]): A 'no data' value to set. Default is 0.
return_windows (Optional[bool]): Whether to return block windows.
warp_mem_limit (Optional[int]): The memory limit (in MB) for the ``rasterio.vrt.WarpedVRT`` function.
num_threads (Optional[int]): The number of warp worker threads.
tap (Optional[bool]): Whether to target align pixels.
kwargs (Optional[dict]): Keyword arguments passed to ``xarray.open_rasterio``.
Returns:
``xarray.DataArray``
"""
ref_kwargs = {
'bounds': None,
'crs': None,
'res': None,
'nodata': nodata,
'warp_mem_limit': warp_mem_limit,
'num_threads': num_threads,
'tap': tap,
'tac': None
}
ref_kwargs_netcdf_stack = ref_kwargs.copy()
ref_kwargs_netcdf_stack['bounds_by'] = 'union'
del ref_kwargs_netcdf_stack['tap']
ref_kwargs = _check_config_globals(filename, 'reference', ref_kwargs)
filenames = None
# Create a list of variables to open
if filename.lower().startswith('netcdf:') and netcdf_vars:
filenames = (f'{filename}:' +
f',{filename}:'.join(netcdf_vars)).split(',')
if filenames:
ref_kwargs_netcdf_stack = _check_config_globals(
filenames[0], 'reference', ref_kwargs_netcdf_stack)
with rio_open(filenames[0]) as src:
tags = src.tags()
else:
ref_kwargs_netcdf_stack = _check_config_globals(
filename, 'reference', ref_kwargs_netcdf_stack)
with rio_open(filename) as src:
tags = src.tags()
@contextlib.contextmanager
def warp_netcdf_vars():
# Warp all images to the same grid.
warped_objects = warp_images(filenames,
resampling=resampling,
**ref_kwargs_netcdf_stack)
yield xr.concat((xr.open_rasterio(wobj, **kwargs)\
.assign_coords(band=[band_names[wi]] if band_names else [netcdf_vars[wi]])
for wi, wobj in enumerate(warped_objects)), dim='band')
with xr.open_rasterio(
warp(filename, resampling=resampling, **ref_kwargs), **
kwargs) if not filenames else warp_netcdf_vars() as src:
if band_names:
if len(band_names) > src.gw.nbands:
src.coords['band'] = band_names[:src.gw.nbands]
else:
src.coords['band'] = band_names
else:
if src.gw.sensor:
if src.gw.sensor not in src.gw.avail_sensors:
if not src.gw.config['ignore_warnings']:
logger.warning(
' The {} sensor is not currently supported.\nChoose from [{}].'
.format(src.gw.sensor,
', '.join(src.gw.avail_sensors)))
else:
new_band_names = list(
src.gw.wavelengths[src.gw.sensor]._fields)
# Avoid nested opens within a `config` context
if len(new_band_names) != len(src.band.values.tolist()):
if not src.gw.config['ignore_warnings']:
logger.warning(
' The new bands, {}, do not match the sensor bands, {}.'
.format(new_band_names,
src.band.values.tolist()))
else:
src.coords['band'] = new_band_names
src.attrs['sensor'] = src.gw.sensor_names[
src.gw.sensor]
if return_windows:
if isinstance(kwargs['chunks'], tuple):
chunksize = kwargs['chunks'][-1]
else:
chunksize = kwargs['chunks']
src.attrs['block_windows'] = get_window_offsets(src.shape[-2],
src.shape[-1],
chunksize,
chunksize,
return_as='list')
src.attrs['filename'] = filename
src.attrs['resampling'] = resampling
if tags:
attrs = src.attrs.copy()
attrs.update(tags)
src = src.assign_attrs(**attrs)
if dtype:
attrs = src.attrs.copy()
return src.astype(dtype).assign_attrs(**attrs)
else:
return src
