def __init__(self, region, sp_res, shapefile=None):
self.country = region
self.shp = Shape(region, shapefile=shapefile)
# countries that cross the international dateline (maybe more!)
if region in ['NZ', 'RS', 'US']:
lonmin, lonmax = dateline_country(region)
else:
lonmin, lonmax = _minmaxcoord(self.shp.bbox[0], self.shp.bbox[2],
sp_res)
latmin, latmax = _minmaxcoord(self.shp.bbox[1], self.shp.bbox[3],
sp_res)
if region in ['NZ', 'US', 'RS']:
lons1 = np.arange(lonmin, 180, sp_res)
lons2 = (np.arange(-180, lonmax + sp_res, sp_res))
lons = np.empty(len(lons1) + len(lons2))
lons[0:len(lons1)] = lons1[:]
lons[len(lons1):] = lons2[:]
else:
lons = np.arange(lonmin, lonmax + sp_res, sp_res)
lats = np.arange(latmin, latmax + sp_res, sp_res)
lon_new, lat_new = _remove_blank_frame(region, lons, lats, shapefile)
lon, lat = np.meshgrid(lon_new, lat_new)
shape = (lon.shape[1], lon.shape[0])
super(ShapeGrid, self).__init__(lon.flatten(), lat.flatten(),
shape=shape)
def validate_prediction(pred):
lons = pred[:,0]
lats = pred[:,1]
data = pred[:,2]
timestamp = np.unique(pred[:,3])[0]
# districts
shapefile = os.path.join('C:\\', 'Users', 'i.pfeil', 'Documents',
'0_IWMI_DATASETS', 'shapefiles', 'IND_adm',
'IND_adm2')
shpfile = Shape(region, shapefile=shapefile)
lon_min, lat_min, lon_max, lat_max = shpfile.bbox
vi_path = "E:\\poets\\RAWDATA\\NDVI\\NDVI_gapfree.nc"
vi_ts = datetime(timestamp.year, timestamp.month, timestamp.day)
vi_data, vi_lons, vi_lats, vi_date = read_img(vi_path, param='NDVI',
lat_min=lat_min, lat_max=lat_max,
lon_min=lon_min, lon_max=lon_max,
timestamp=vi_ts)
vi_lon, vi_lat = np.meshgrid(vi_lons, vi_lats)
vi_lon = vi_lon.flatten()
vi_lat = vi_lat.flatten()
scatter_subplots(lons, lats, data, 200,
vi_lon, vi_lat, vi_data, 200,
title1=str(vi_date)+' - simulated',
title2=str(vi_date)+' - orig. data')
return pred
def test_Shape(self):
# test default shapefile
shp = Shape(self.region)
assert shp.code == self.region
assert shp.name == self.name
assert shp.bbox == (9.5335693359375, 46.407493591308594,
17.166385650634766, 49.01874542236328)
# test custom shapefile
cshp = Shape('NOE', shapefile=self.shapefile)
assert cshp.code == 'NOE'
assert cshp.name == 'NOE'
assert cshp.bbox == (14.455012306812042, 47.422602087551724,
17.07012761362633, 49.021162769139316)
# test if error is raised
self.assertRaises(FipsError, Shape, 'XYZ')
def bounds(country, shapefile=None):
"""
Returns the bounding box, center coordinates and zoom level
of a shape for web overlay purposes.
Parameters
----------
country : str
FIPS country code (https://en.wikipedia.org/wiki/FIPS_country_code)
shapefile : str, optional
Paht to a custom shapefile.
Returns
-------
lon_min : int
Minimum longitude.
lon_max : int
Maximum longitude.
lat_min : int
Minimum latitude.
lat_max : int
Maximum latitude.
c_lat : int
Center latidute of image.
c_lon : int
Center longitude of image.
zoom : int
Zoom level for openlayers.
"""
shp = Shape(country, shapefile)
lon_min = shp.bbox[0]
lon_max = shp.bbox[2]
lat_min = shp.bbox[1]
lat_max = shp.bbox[3]
e_lon = lon_max - lon_min
e_lat = lat_max - lat_min
c_lon = lon_min + e_lon / 2
c_lat = lat_min + e_lat / 2
zoom = 0
i = 1024 # To be replaced with the width of the map container!
while i / 2 > e_lon:
zoom += 1
i = i / 2
return lon_min, lon_max, lat_min, lat_max, c_lat, c_lon, zoom
def get_geojson(region):
"""
Gets list of coordinates from polygon of region.
Parameters
----------
region : str
Region to get coordinates from.
Returns
-------
coordinates : list
"""
shape = Shape(region, p.shapefile).polygon
return jsonify(mapping(shape))
def resample_to_shape(source_file,
region,
sp_res,
grid,
prefix=None,
nan_value=None,
dest_nan_value=None,
variables=None,
shapefile=None):
"""
Resamples images and clips country boundaries
Parameters
----------
source_file : str
Path to source file.
region : str
Identifier of the region in the shapefile. If the default shapefile is
used, this would be the FIPS country code.
sp_res : int or float
Spatial resolution of the shape-grid.
grid : poets.grid.RegularGrid or poets.grid.ShapeGrid
Grid to resample data to.
prefix : str, optional
Prefix for the variable in the NetCDF file, should be name of source
nan_value : int, float, optional
Not a number value of the original data as given by the data provider
dest_nan_value : int or float, optional
NaN value used in the final NetCDF file.
variables : list of str, optional
Variables to resample from original file.
shapefile : str, optional
Path to shape file, uses "world country admin boundary shapefile" by
default.
Returns
-------
res_data : dict of numpy.arrays
resampled image
dest_lon : numpy.array
longitudes of the points in the resampled image
dest_lat : numpy.array
latitudes of the points in the resampled image
gpis : numpy.array
grid point indices
timestamp : datetime.date
date of the image
metadata : dict
Metadata derived from input file.
"""
if prefix is not None:
prefix += '_'
fileExtension = os.path.splitext(source_file)[1].lower()
if region == 'global':
lon_min = -180
lon_max = 180
lat_min = -90
lat_max = 90
else:
shp = Shape(region, shapefile)
lon_min = shp.bbox[0]
lon_max = shp.bbox[2]
lat_min = shp.bbox[1]
lat_max = shp.bbox[3]
if fileExtension in ['.nc', '.nc3', '.nc4']:
data_src, lon, lat, timestamp, metadata = nc.read_image(
source_file, variables)
if (lon_min >= lon.max() or lon_max <= lon.min()
or lat_max <= lat.min() or lat_min >= lat.max()):
return "No data"
data, src_lon, src_lat = nc.clip_bbox(data_src, lon, lat, lon_min,
lat_min, lon_max, lat_max)
elif fileExtension in ['.h5']:
data_src, lon, lat, timestamp, metadata = h5.read_image(
source_file, variables)
if (lon_min >= lon.max() or lon_max <= lon.min()
or lat_max <= lat.min() or lat_min >= lat.max()):
return "No data"
data, src_lon, src_lat = nc.clip_bbox(data_src, lon, lat, lon_min,
lat_min, lon_max, lat_max)
elif fileExtension in imgfiletypes:
data, src_lon, src_lat, timestamp, metadata = bbox_img(
source_file, region, fileExtension, shapefile)
if nan_value is not None:
for key in data.keys():
data[key] = np.ma.array(data[key], mask=(data[key] == nan_value))
src_lon, src_lat = np.meshgrid(src_lon, src_lat)
lons = grid.arrlon[0:grid.shape[0]]
dest_lon, dest_lat = np.meshgrid(lons, np.unique(grid.arrlat)[::-1])
gpis = grid.get_bbox_grid_points(grid.arrlat.min(), grid.arrlat.max(),
grid.arrlon.min(), grid.arrlon.max())
search_rad = 180000 * sp_res
data = resample.resample_to_grid(data,
src_lon,
src_lat,
dest_lon,
dest_lat,
search_rad=search_rad)
res_data = {}
path = []
if region != 'global':
_, _, multipoly = shp._get_shape()
for ring in multipoly:
poly_verts = list(ring.exterior.coords)
path.append(matplotlib.path.Path(poly_verts))
coords = [grid.arrlon, grid.arrlat[::-1]]
coords2 = np.zeros((len(coords[0]), 2))
for idx in range(0, len(coords[0])):
coords2[idx] = [coords[0][idx], coords[1][idx]]
mask_old = path[0].contains_points(coords2)
for key in data.keys():
if variables is not None:
if key not in variables:
del metadata[key]
continue
if region != 'global':
for ring in path:
mask_new = (ring.contains_points(coords2))
mask_rev = scipy.logical_or(mask_old, mask_new)
mask_old = mask_rev
mask_rev = mask_rev.reshape(dest_lon.shape)
mask = np.invert(mask_rev)
mask[data[key].mask == True] = True
else:
mask = data[key].mask
if prefix is None:
var = key
else:
var = prefix + key
if metadata is not None:
metadata[var] = metadata[key]
if var != key:
del metadata[key]
res_data[var] = np.ma.masked_array(data[key],
mask=np.copy(mask),
fill_value=dest_nan_value)
dat = np.copy(res_data[var].data)
dat[mask == True] = dest_nan_value
res_data[var] = np.ma.masked_array(dat,
mask=np.copy(mask),
fill_value=dest_nan_value)
return res_data, dest_lon, dest_lat, gpis, timestamp, metadata
def _remove_blank_frame(region, lons, lats, shapefile=None):
"""Removes longitudes and latitudes without points in region shape.
Parameters
----------
region : str
FIPS country code (https://en.wikipedia.org/wiki/FIPS_country_code)
lons : numpy.ndarray
Array with longitudes.
lats : numpy.ndarray
Array with latitudes.
Returns
--------
lon_new : list of floats
Updated list of longitudes.
lat_new : list of floats
Updated list of latitudes.
"""
shp = Shape(region, shapefile)
poly = shp.polygon
del_lons = []
del_lats = []
# left boundary check
for i, x in enumerate(lons[:(lons.size / 2)]):
checksum = 0
for y in lats:
p = Point(x, y)
if not p.within(poly):
checksum += 1
if checksum == lats.size:
del_lons.append(i)
else:
break
# right boundary check
for i, x in enumerate(lons[::-1][:(lons.size / 2)]):
checksum = 0
for y in lats:
p = Point(x, y)
if not p.within(poly):
checksum += 1
if checksum == lats.size:
del_lons.append(lons.size - 1 - i)
else:
break
# bottom boundary check
for i, y in enumerate(lats[:(lats.size / 2)]):
checksum = 0
for x in lons:
p = Point(x, y)
if not p.within(poly):
checksum += 1
if checksum == lons.size:
del_lats.append(i)
else:
break
# top boundary check
for i, y in enumerate(lats[::-1][:(lats.size / 2)]):
checksum = 0
for x in lons:
p = Point(x, y)
if not p.within(poly):
checksum += 1
if checksum == lons.size:
del_lats.append(lats.size - 1 - i)
else:
break
lon_new = lons.tolist()
lat_new = lats.tolist()
for i in del_lons:
if lons[i] in lon_new:
lon_new.remove(lons[i])
for i in del_lats:
if lats[i] in lat_new:
lat_new.remove(lats[i])
return lon_new, lat_new
def resample_to_shape(source_file, region, sp_res, grid, prefix=None,
nan_value=None, dest_nan_value=None, variables=None,
shapefile=None):
"""
Resamples images and clips country boundaries
Parameters
----------
source_file : str
Path to source file.
region : str
Identifier of the region in the shapefile. If the default shapefile is
used, this would be the FIPS country code.
sp_res : int or float
Spatial resolution of the shape-grid.
grid : poets.grid.RegularGrid or poets.grid.ShapeGrid
Grid to resample data to.
prefix : str, optional
Prefix for the variable in the NetCDF file, should be name of source
nan_value : int, float, optional
Not a number value of the original data as given by the data provider
dest_nan_value : int or float, optional
NaN value used in the final NetCDF file.
variables : list of str, optional
Variables to resample from original file.
shapefile : str, optional
Path to shape file, uses "world country admin boundary shapefile" by
default.
Returns
-------
res_data : dict of numpy.arrays
resampled image
dest_lon : numpy.array
longitudes of the points in the resampled image
dest_lat : numpy.array
latitudes of the points in the resampled image
gpis : numpy.array
grid point indices
timestamp : datetime.date
date of the image
metadata : dict
Metadata derived from input file.
"""
if prefix is not None:
prefix += '_'
fileExtension = os.path.splitext(source_file)[1].lower()
if region == 'global':
lon_min = -180
lon_max = 180
lat_min = -90
lat_max = 90
else:
shp = Shape(region, shapefile)
lon_min = shp.bbox[0]
lon_max = shp.bbox[2]
lat_min = shp.bbox[1]
lat_max = shp.bbox[3]
if fileExtension in ['.nc', '.nc3', '.nc4']:
data_src, lon, lat, timestamp, metadata = nc.read_image(source_file,
variables)
if (lon_min >= lon.max() or lon_max <= lon.min() or
lat_max <= lat.min() or lat_min >= lat.max()):
return "No data"
data, src_lon, src_lat = nc.clip_bbox(data_src, lon, lat, lon_min,
lat_min, lon_max, lat_max)
elif fileExtension in ['.h5']:
data_src, lon, lat, timestamp, metadata = h5.read_image(source_file,
variables)
if (lon_min >= lon.max() or lon_max <= lon.min() or
lat_max <= lat.min() or lat_min >= lat.max()):
return "No data"
data, src_lon, src_lat = nc.clip_bbox(data_src, lon, lat, lon_min,
lat_min, lon_max, lat_max)
elif fileExtension in imgfiletypes:
data, src_lon, src_lat, timestamp, metadata = bbox_img(source_file,
region,
fileExtension,
shapefile)
if nan_value is not None:
for key in data.keys():
data[key] = np.ma.array(data[key], mask=(data[key] == nan_value))
src_lon, src_lat = np.meshgrid(src_lon, src_lat)
lons = grid.arrlon[0:grid.shape[0]]
dest_lon, dest_lat = np.meshgrid(lons, np.unique(grid.arrlat)[::-1])
gpis = grid.get_bbox_grid_points(grid.arrlat.min(), grid.arrlat.max(),
grid.arrlon.min(), grid.arrlon.max())
search_rad = 180000 * sp_res
data = resample.resample_to_grid(data, src_lon, src_lat, dest_lon,
dest_lat, search_rad=search_rad)
res_data = {}
path = []
if region != 'global':
_, _, multipoly = shp._get_shape()
for ring in multipoly:
poly_verts = list(ring.exterior.coords)
path.append(matplotlib.path.Path(poly_verts))
coords = [grid.arrlon, grid.arrlat[::-1]]
coords2 = np.zeros((len(coords[0]), 2))
for idx in range(0, len(coords[0])):
coords2[idx] = [coords[0][idx], coords[1][idx]]
mask_old = path[0].contains_points(coords2)
for key in data.keys():
if variables is not None:
if key not in variables:
del metadata[key]
continue
if region != 'global':
for ring in path:
mask_new = (ring.contains_points(coords2))
mask_rev = scipy.logical_or(mask_old, mask_new)
mask_old = mask_rev
mask_rev = mask_rev.reshape(dest_lon.shape)
mask = np.invert(mask_rev)
mask[data[key].mask == True] = True
else:
mask = data[key].mask
if prefix is None:
var = key
else:
var = prefix + key
if metadata is not None:
metadata[var] = metadata[key]
if var != key:
del metadata[key]
res_data[var] = np.ma.masked_array(data[key], mask=np.copy(mask),
fill_value=dest_nan_value)
dat = np.copy(res_data[var].data)
dat[mask == True] = dest_nan_value
res_data[var] = np.ma.masked_array(dat, mask=np.copy(mask),
fill_value=dest_nan_value)
return res_data, dest_lon, dest_lat, gpis, timestamp, metadata
def bbox_img(source_file, region, fileExtension, shapefile=None):
"""
Clips bounding box out of image file and returns data as numpy.ndarray
Parameters
----------
source_file : str
Path to source file.
region : str
Identifier of the region in the shapefile. If the default shapefile is
used, this would be the FIPS country code.
fileExtension : str
Filetype (e.g. png, tif).
shapefile : str, optional
Path to shape file, uses "world country admin boundary shapefile" by
default.
Returns
-------
data : dict of numpy.arrays
Clipped image (grey values).
lon_new : numpy.array
Longitudes of the clipped image.
lat_new : numpy.array
Latitudes of the clipped image.
timestamp : datetime.date
Timestamp of the image.
metadata : dict of strings
Metadata from source netCDF file.
"""
orig_img = Image.open(source_file)
lon_min_src, lat_min_src, lon_max_src, lat_max_src = \
get_layer_extent(source_file)
if region == 'global':
lon_min = -180
lon_max = 180
lat_min = -90
lat_max = 90
else:
shp = Shape(region, shapefile)
lon_min = shp.bbox[0]
lon_max = shp.bbox[2]
lat_min = shp.bbox[1]
lat_max = shp.bbox[3]
d = lon_max - lon_min
# countries that cross the international dateline (maybe more!)
if region in ['NZ', 'RS', 'US']:
lon_min, lon_max = dateline_country(region)
# get 2 pairs of points (upper left, lower right of bbox)
if d > 350 and region not in ['AY', 'global']:
if fileExtension in ['.tif', '.tiff', '.TIF', '.TIFF']:
if (round(lon_max_src - lon_min_src) == 360.0
and round(lat_max_src - lat_min_src) == 180.0):
orig_img = rearrange_img(orig_img)
row_min, col_min = lonlat2px_rearr(orig_img, lon_min, lat_max)
row_max, col_max = lonlat2px_rearr(orig_img, lon_max, lat_min)
img = orig_img.crop(
(int(math.floor(col_min)), int(math.floor(row_min)),
int(math.ceil(col_max)), int(math.ceil(row_max))))
else:
print 'Rearranging is only possible for global imagefiles.'
return
else:
orig_img = rearrange_img(orig_img)
row_min, col_min = lonlat2px_rearr(orig_img, lon_min, lat_max)
row_max, col_max = lonlat2px_rearr(orig_img, lon_max, lat_min)
img = orig_img.crop(
(int(math.floor(col_min)), int(math.floor(row_min)),
int(math.ceil(col_max)), int(math.ceil(row_max))))
elif fileExtension in ['.tif', '.tiff', '.TIF', '.TIFF']:
row_min, col_min = lonlat2px_gt(orig_img, lon_min, lat_max,
lon_min_src, lat_min_src, lon_max_src,
lat_max_src)
row_max, col_max = lonlat2px_gt(orig_img, lon_max, lat_min,
lon_min_src, lat_min_src, lon_max_src,
lat_max_src)
# crop image
img = orig_img.crop(
(int(math.floor(col_min)), int(math.floor(row_min)),
int(math.ceil(col_max)), int(math.ceil(row_max))))
else:
row_min, col_min = lonlat2px(orig_img, lon_min, lat_max)
row_max, col_max = lonlat2px(orig_img, lon_max, lat_min)
# crop image
img = orig_img.crop(
(int(math.floor(col_min)), int(math.floor(row_min)),
int(math.ceil(col_max)), int(math.ceil(row_max))))
# get data values from image
data = {'dataset': np.array(img)}
# lon_new, lat_new
lon_px = np.arange(int(math.floor(col_min)), int(math.ceil(col_max)))
lat_px = np.arange(int(math.floor(row_min)), int(math.ceil(row_max)))
if region in ['NZ', 'RS', 'US']:
lon_new, lat_new = px2lonlat_rearr(orig_img, lon_px, lat_px)
elif fileExtension in ['.tif', '.tiff', '.TIF', '.TIFF']:
lon_new, lat_new = px2lonlat_gt(orig_img, lon_px, lat_px, lon_min_src,
lat_min_src, lon_max_src, lat_max_src)
else:
lon_new, lat_new = px2lonlat(orig_img, lon_px, lat_px)
# timestamp
timestamp = None
# metadata
metadata = None
# move coordinates to pixel center
if lon_new.size > 2:
lon_new += (lon_new[1] - lon_new[0]) / 2
if lat_new.size > 2:
lat_new += (lat_new[1] - lat_new[0]) / 2
return data, lon_new, lat_new, timestamp, metadata
def start_pred(paths, region, pred_date, vi_str='NDVI',
t_val='SWI_040', monthly=False, spatial_res=0.1):
if spatial_res == 0.1:
with Dataset(paths['SWI'], 'r') as ncfile:
res_lons = ncfile.variables['lon'][:]
res_lats = ncfile.variables['lat'][:]
elif spatial_res == 500:
with Dataset(paths['NDVI'], 'r') as ncfile:
res_lons = ncfile.variables['lon'][:]
res_lats = ncfile.variables['lat'][:]
with Dataset(paths['lc']) as ncfile:
lccs = ncfile.variables['lccs_class'][:]
lc_lons = ncfile.variables['lon'][:]
lc_lats = ncfile.variables['lat'][:]
#===========================================================================
# # achtung pc haengt sich eine zeit lang auf
# lc_lons, lc_lats = np.meshgrid(lc_lons, lc_lats)
# lc_lons = lc_lons.flatten()
# lc_lats = lc_lats.flatten()
# scatterplot(lc_lons, lc_lats, lccs, discrete=False, vmin=-128, vmax=128)
#===========================================================================
# districts
shapefile = os.path.join('C:\\', 'Users', 'i.pfeil', 'Documents',
'0_IWMI_DATASETS', 'shapefiles', 'IND_adm',
'IND_adm2')
shpfile = Shape(region, shapefile=shapefile)
lon_min, lat_min, lon_max, lat_max = shpfile.bbox
lons = res_lons[np.where((res_lons>=lon_min) & (res_lons<=lon_max))]
lats = res_lats[np.where((res_lats>=lat_min) & (res_lats<=lat_max))]
start_date = datetime(2007,7,1)
end_date = datetime(2015,7,1)
results = []
results2 = []
results3 = []
for lon in lons:
for lat in lats:
print lon, lat
if round(lon,2) == 76.35 and round(lat,2) == 19.55:
print 'danger'
nearest_lon = find_nearest(lc_lons, lon)
nearest_lat = find_nearest(lc_lats, lat)
lc = lccs[nearest_lat, nearest_lon]
if (lc == -66) or (lc == -46) or (lc == -36) or (lc == 0):
# urban | water | snow and ice | no data
print 'lc mask'
continue
swi_path = paths['SWI']
vi_path = paths[vi_str]
swi_list = [t_val]
swi_df = read_ts(swi_path, lon=lon, lat=lat, params=swi_list,
start_date=start_date, end_date=end_date)
# read vi and scale from 0 to 100 (before 0 to 250)
vi_all = read_ts(vi_path, lon=lon, lat=lat, params=vi_str,
start_date=start_date, end_date=end_date)
#vi_all[vi_str][np.where(vi_all==-99)[0]] = np.NaN
#vi_all = vi_all*100/250
vi = vi_all[:pred_date]
vi_min = np.nanmin(vi)
vi_max = np.nanmax(vi)
vi = rescale_peng(vi, vi_min, vi_max)
swi_all = swi_df[t_val]
swi = swi_all[:pred_date]
swi = rescale_peng(swi, np.nanmin(swi), np.nanmax(swi))
# resample monthly
if monthly:
swi = swi.resample("M").mean()
vi = vi.resample("M").mean()
# calculate differences between VIs of consecutive months
dvi = np.ediff1d(vi, to_end=np.NaN)
vi['D_VI'] = pd.Series(dvi, index=vi.index)
matched_data = temp_match.matching(swi, vi)
kd = zribi_kd(swi, vi, matched_data)
results, results2, results3 = zribi_sim(lon, lat, swi, vi,
matched_data, kd, vi_min, vi_max,
results=results, results2=results2,
results3=results3)
np.save('C:\\Users\\i.pfeil\\Desktop\\veg_prediction\\results.npy', results)
np.save('C:\\Users\\i.pfeil\\Desktop\\veg_prediction\\results2.npy', results2)
np.save('C:\\Users\\i.pfeil\\Desktop\\veg_prediction\\results3.npy', results3)