class ProjectFlood(object):
proj_dir = "c:/data/projects/"
shape_countries = "c:/data/input_data/gaul_2014_2008_2/gaul_wfp.shp"
pop_distr = "c:/data/input_data/population/popmap10.tif"
driver = ogr.GetDriverByName("ESRI Shapefile")
shapefile = driver.Open(shape_countries)
layer = shapefile.GetLayer()
historical_table = "c:/data/input_data/historical_data/floods - refine.csv"
def __init__(self, paese, admin):
self.paese = paese
self.admin = admin
self.dirOut = HazardAssessment.proj_dir + paese + "/" + admin + "/out/"
def load_shapefile(src_path, pg_host, pg_db, pg_user, pg_pwd):
connection = psycopg2.connect(host=pg_host,
database=pg_db,
user=pg_user,
password=pg_pwd)
cursor = connection.cursor()
ddl = """
DROP TABLE IF EXISTS public.borders;
CREATE TABLE public.borders
(
id SERIAL NOT NULL
PRIMARY KEY,
name VARCHAR NOT NULL,
iso_code VARCHAR NOT NULL,
outline GEOGRAPHY NOT NULL
);
CREATE INDEX border_index
ON public.borders
USING GIST(outline);
"""
dml = """
INSERT INTO public.borders (name, iso_code, outline)
VALUES (%s, %s, ST_GeogFromText(%s))
"""
cursor.execute(ddl)
connection.commit()
shapefile = ogr.Open(src_path)
layer = shapefile.GetLayer(0)
for i in range(layer.GetFeatureCount()):
feature = layer.GetFeature(i)
name = feature.GetField("NAME")
iso_code = feature.GetField("ISO3")
geometry = feature.GetGeometryRef()
wkt = geometry.ExportToWkt()
cursor.execute(dml, (name, iso_code, wkt))
connection.commit()
def field_exists(field_name, shp_path):
"""
This function returns whether the specified field exists or not in the shapefile linked by a path.
:param field_name: Name of the field to be checked for.
:type field_name: str
:param shp_path: Path to the shapefile.
:type shp_path: str
:return: ``True`` if it exists or ``False`` if it doesn't exist.
:rtype: bool
"""
shp = ogr.Open(shp_path, 0)
lyr = shp.GetLayer()
lyr_dfn = lyr.GetLayerDefn()
exists = False
for i in range(lyr_dfn.GetFieldCount()):
exists = exists or (field_name == lyr_dfn.GetFieldDefn(i).GetName())
return exists
def catchment_slice(X, catchment_filenames):
"""Slice the output array in x- and y-direction in order to only save the
extent of one or multiple catchments. Currently, this tool only works for
the KNMI radar datasets.
Parameters
-------------
X : 3D-array, which is the output of the steps.py ensemble nowcast per time
step.The array consists of:
1. n ensemble members
2. n rows
3. n cols
catchment_filenames: A list with the location (directory + filename) of
catchment shapefiles. The shapefiles will be used and rasterized in a
loop.
metadata: The metadata of the input radar images.
Returns
-------------
array_out: 4D-array, though similar to the input, but with sliced rows and cols.
Hence, this array consists of:
1. n catchments
2. n ensemble members
3. n rows (sliced)
4. n cols (sliced)
metadata_out: The metadata, conform the input metadata, but for the sliced
arrays. The output is an array consisting of n metadata-files.
"""
##########
# Initial settings
##########
# Set the info from the NL_RAD data
xmin_nlrad = 0
xmax_nlrad = 700
ymin_nlrad = -4415
ymax_nlrad = -3650
###########
# Loop to rasterize the shapefile and then slice the radar array with the
# extent of the rasterized shapefile.
###########
# This is going to be the output
array_out = []
for i in range(0, len(catchment_filenames)):
#########################################################
# Time to read the shapefile
#########################################################
filename = catchment_filenames[i]
# set file names in order to obtain the reprojected shapefile, which
# was made with the catchment_medata functionality.
dirname = os.path.dirname(filename)
basename = os.path.basename(filename)
basenametxt = os.path.splitext(basename)[0]
outfile = os.path.join(dirname, basenametxt + '_Reprojected.shp')
#########################################################
# Define the new, reprojected shapefile as sfnew
# We are going to rasterize this shapefile, since the resulting radar raster (after
# rasterizing the hdf5-dataset) should be exactly the same as this raster of this shapefile
#########################################################
# sfnew = shapefile.Reader(outfile)
# 1. Define pixel_size and NoData value of new raster
NoData_value = -9999
# 2. Open Shapefile
driver = ogr.GetDriverByName('ESRI Shapefile')
shapefile = driver.Open(outfile)
source_layer = shapefile.GetLayer()
# Obtain xmin, xmax, ymin and ymax as floats
x_min, x_max, y_min, y_max = source_layer.GetExtent()
# 3. Round the values for xmin, xmax, ymin and ymax (make a rough rounding)
# in order to not get a too small extent. Finally, make integer values of it.
xmin = int(round(x_min, 0) - 1)
xmax = int(round(x_max, 0) + 1)
ymin = int(round(y_min, 0) - 1)
ymax = int(round(y_max, 0) + 1)
# 4. Slice the array with the known coordinates - which equal the col and row number!
xslice_min = xmin
xslice_max = xmax
yslice_max = (ymin - ymax_nlrad
) * -1 # + 3650 in order to compensate for the 3650
yslice_min = (
ymax - ymax_nlrad
) * -1 # ymin and ymax are flipped, because the indexation is flipped as well
array_new = X[:, yslice_min:yslice_max, xslice_min:xslice_max]
array_out.append(array_new)
array_new = None
xmin = None
xmax = None
ymin = None
ymax = None
xslice_min = None
xslice_max = None
yslice_min = None
yslice_max = None
return array_out
def catchment_metadata(catchment_filenames, metadata):
"""Obtain new metadata files per given catchment shapefile.
Currently, this tool only works for the KNMI radar datasets.
Parameters
-------------
catchment_filenames: A list with the location (directory + filename) of
catchment shapefiles. The shapefiles will be used and rasterized in a
loop.
metadata: The metadata of the input radar images.
Returns
-------------
metadata_out: The metadata, conform the input metadata, but for the sliced
arrays. The output is an array consisting of n metadata-files.
"""
##########
# Initial settings
##########
# Set the info from the NL_RAD data
xmin_nlrad = 0
xmax_nlrad = 700
ymin_nlrad = -4415
ymax_nlrad = -3650
###########
# Loop to rasterize the shapefile and then slice the radar array with the
# extent of the rasterized shapefile.
###########
# This is going to be the output
metadata_out = []
for i in range(0, len(catchment_filenames)):
#########################################################
# Time to read the shapefile
#########################################################
filename = catchment_filenames[i]
# sf = shapefile.Reader(filename)
#########################################################
# Get the projection of the shapefiles
##########################################################
driver = ogr.GetDriverByName('ESRI Shapefile')
dataset = driver.Open(filename)
# from Layer
layer = dataset.GetLayer()
spatialRef = layer.GetSpatialRef()
# from Geometry
feature = layer.GetNextFeature()
geom = feature.GetGeometryRef()
spatialRef = geom.GetSpatialReference()
# print("The spatial projection is: %s" %spatialRef)
#########################################################
# Reproject the geometry of the shapefile
#########################################################
# set file names
infile = filename
dirname = os.path.dirname(filename)
basename = os.path.basename(filename)
basenametxt = os.path.splitext(basename)[0]
outfile = os.path.join(dirname, basenametxt + '_Reprojected.shp')
# The projection will be based on the projection of the given data set
ShapefileProj = '+proj=stere +lat_0=90 +lon_0=0.0 +lat_ts=60.0 +a=6378.137 +b=6356.752 +x_0=0 +y_0=0'
subprocess.call(['ogr2ogr', '-t_srs', ShapefileProj, outfile, infile])
#########################################################
# Define the new, reprojected shapefile as sfnew
# We are going to 'rasterize' this shapefile, since the resulting radar raster (after
# rasterizing the hdf5-dataset) should be exactly the same as this raster of this shapefile
#########################################################
# sfnew = shapefile.Reader(outfile)
# 1. Define pixel_size and NoData value of new raster
NoData_value = -9999
# 2. Open Shapefile
driver = ogr.GetDriverByName('ESRI Shapefile')
shapefile = driver.Open(outfile)
source_layer = shapefile.GetLayer()
# Obtain xmin, xmax, ymin and ymax as floats
x_min, x_max, y_min, y_max = source_layer.GetExtent()
# 3. Round the values for xmin, xmax, ymin and ymax (make a rough rounding)
# in order to not get a too small extent. Finally, make integer values of it.
xmin = int(round(x_min, 0) - 1)
xmax = int(round(x_max, 0) + 1)
ymin = int(round(y_min, 0) - 1)
ymax = int(round(y_max, 0) + 1)
x_res = 1
y_res = 1
cols = int((xmax - xmin) / x_res)
rows = int((ymax - ymin) / y_res)
# 4. Fill in the new metadata
metadata_new = metadata.copy()
metadata_new["shape"] = (rows, cols)
metadata_new["x1"] = xmin
metadata_new["x2"] = xmax
metadata_new["y1"] = ymax
metadata_new["y2"] = ymin
metadata_out.append(metadata_new)
xmin = None
xmax = None
ymin = None
ymax = None
return metadata_out
def catchment_slice_mpi(X, catchment_shapes):
"""Slice the output array in x- and y-direction in order to only save the
extent of one or multiple catchments. Currently, this tool only works for
the KNMI radar datasets.
Parameters
-------------
X : 3D-array, which is the output of the steps.py ensemble nowcast per time
step.The array consists of:
1. n ensemble members
2. n rows
3. n cols
catchment_shapes: A list with already opened shapefiles of the catchments.
The shapefiles will be used to get the extent of the radar image. Note
that the shapefiles have to be reprojected in advance (this in contrast
to the original serial catchment_slice.py script!).
metadata: The metadata of the input radar images.
Returns
-------------
array_out: 4D-array, though similar to the input, but with sliced rows and cols.
Hence, this array consists of:
1. n catchments
2. n ensemble members
3. n rows (sliced)
4. n cols (sliced)
metadata_out: The metadata, conform the input metadata, but for the sliced
arrays. The output is an array consisting of n metadata-files.
"""
##########
# Initial settings
##########
# Set the info from the NL_RAD data
xmin_nlrad = 0
xmax_nlrad = 700
ymin_nlrad = -4415
ymax_nlrad = -3650
###########
# Loop to rasterize the shapefile and then slice the radar array with the
# extent of the rasterized shapefile.
###########
# This is going to be the output
array_out = []
for i in range(0, len(catchment_shapes)):
# 1. Define pixel_size and NoData value of new raster
NoData_value = -9999
# 2. Get the shapefile
# driver = ogr.GetDriverByName('ESRI Shapefile')
shapefile = catchment_shapes[i]
source_layer = shapefile.GetLayer()
# Obtain xmin, xmax, ymin and ymax as floats
x_min, x_max, y_min, y_max = source_layer.GetExtent()
# 3. Round the values for xmin, xmax, ymin and ymax (make a rough rounding)
# in order to not get a too small extent. Finally, make integer values of it.
xmin = int(round(x_min, 0) - 1)
xmax = int(round(x_max, 0) + 1)
ymin = int(round(y_min, 0) - 1)
ymax = int(round(y_max, 0) + 1)
# 4. Slice the array with the known coordinates - which equal the col and row number!
xslice_min = xmin
xslice_max = xmax
yslice_max = (ymin - ymax_nlrad
) * -1 # + 3650 in order to compensate for the 3650
yslice_min = (
ymax - ymax_nlrad
) * -1 # ymin and ymax are flipped, because the indexation is flipped as well
array_new = X[:, yslice_min:yslice_max, xslice_min:xslice_max]
array_out.append(array_new)
array_new = None
xmin = None
xmax = None
ymin = None
ymax = None
xslice_min = None
xslice_max = None
yslice_min = None
yslice_max = None
return array_out
def catchment_metadata_mpi(catchment_shapes, metadata):
"""Obtain new metadata files per given catchment shapefile.
Currently, this tool only works for the KNMI radar datasets.
Parameters
-------------
catchment_filenames: A list with the location (directory + filename) of
catchment shapefiles. The shapefiles will be used and rasterized in a
loop.
metadata: The metadata of the input radar images.
Returns
-------------
metadata_out: The metadata, conform the input metadata, but for the sliced
arrays. The output is an array consisting of n metadata-files.
"""
##########
# Initial settings
##########
# Set the info from the NL_RAD data
xmin_nlrad = 0
xmax_nlrad = 700
ymin_nlrad = -4415
ymax_nlrad = -3650
###########
# Loop to rasterize the shapefile and then slice the radar array with the
# extent of the rasterized shapefile.
###########
# This is going to be the output
metadata_out = []
for i in range(0, len(catchment_shapes)):
shapefile = catchment_shapes[i]
source_layer = shapefile.GetLayer()
# Obtain xmin, xmax, ymin and ymax as floats
x_min, x_max, y_min, y_max = source_layer.GetExtent()
# 3. Round the values for xmin, xmax, ymin and ymax (make a rough rounding)
# in order to not get a too small extent. Finally, make integer values of it.
xmin = int(round(x_min, 0) - 1)
xmax = int(round(x_max, 0) + 1)
ymin = int(round(y_min, 0) - 1)
ymax = int(round(y_max, 0) + 1)
x_res = 1
y_res = 1
cols = int((xmax - xmin) / x_res)
rows = int((ymax - ymin) / y_res)
# 4. Fill in the new metadata
metadata_new = metadata.copy()
metadata_new["shape"] = (rows, cols)
metadata_new["x1"] = xmin
metadata_new["x2"] = xmax
metadata_new["y1"] = ymax
metadata_new["y2"] = ymin
metadata_out.append(metadata_new)
xmin = None
xmax = None
ymin = None
ymax = None
return metadata_out
class UtilitieSparc(object):
proj_dir = os.getcwd() + "/projects/"
driver = ogr.GetDriverByName("ESRI Shapefile")
#DA MAIN
shape_countries = "input_data/gaul/gaul_wfp_iso.shp"
campo_nome_paese = "ADM0_NAME"
campo_iso_paese = "ADM0_CODE"
campo_nome_admin = "ADM2_NAME"
campo_iso_admin = "ADM2_CODE"
shapefile = driver.Open(shape_countries)
layer = shapefile.GetLayer()
nome_paese = ""
cod_paese = ""
def __init__(self, paese, admin):
self.dati_per_plot = {}
self.dati_per_prob = {}
self.paese = paese
self.admin = admin
self.dirOut = UtilitieSparc.proj_dir + paese + "/" + admin + "/"
def lista_admin0(self):
numFeatures = self.layer.GetFeatureCount()
lista_stati = []
lista_iso = []
for featureNum in range(numFeatures):
feature = self.layer.GetFeature(featureNum)
nome_paese = feature.GetField(self.campo_nome_paese)
code_paese = feature.GetField(self.campo_iso_paese)
lista_stati.append(nome_paese)
lista_iso.append(code_paese)
seen = set()
seen_add = seen.add
lista_pulita = [
x for x in lista_stati if not (x in seen or seen_add(x))
]
lista_admin0 = sorted(lista_pulita)
return tuple(lista_admin0)
def lista_admin2(self, country):
country_capitalized = country.capitalize()
self.layer.SetAttributeFilter(self.campo_nome_paese + " = '" +
country_capitalized + "'")
listone = {}
lista_iso = []
lista_clean = []
lista_admin2 = []
for feature in self.layer:
cod_admin = feature.GetField(self.campo_iso_admin)
nome_zozzo = feature.GetField(self.campo_nome_admin)
unicode_zozzo = nome_zozzo.decode('utf-8')
nome_per_combo = unicodedata.normalize('NFKD', unicode_zozzo)
no_dash = re.sub('-', '_', nome_zozzo)
no_space = re.sub(' ', '', no_dash)
no_slash = re.sub('/', '_', no_space)
no_apice = re.sub('\'', '', no_slash)
no_bad_char = re.sub(r'-/\([^)]*\)', '', no_apice)
unicode_pulito = no_bad_char.decode('utf-8')
nome_pulito = unicodedata.normalize('NFKD', unicode_pulito).encode(
'ascii', 'ignore')
lista_iso.append(cod_admin)
lista_clean.append(nome_pulito)
lista_admin2.append(nome_per_combo)
for i in range(len(lista_iso)):
listone[lista_iso[i]] = {
'name_orig': lista_admin2[i],
'name_clean': lista_clean[i]
}
return lista_admin2, listone
def creazione_struttura(self, admin_global):
# Check in data structures exists and in case not create the directory named
# after the country and all the directories
UtilitieSparc.proj_dir
os.chdir(UtilitieSparc.proj_dir)
country_low = str(self.paese).lower()
if os.path.exists(country_low):
os.chdir(UtilitieSparc.proj_dir + country_low)
admin_low = str(self.admin).lower()
if os.path.exists(admin_low):
pass
else:
os.mkdir(admin_low)
else:
os.chdir(UtilitieSparc.proj_dir)
os.mkdir(country_low)
os.chdir(UtilitieSparc.proj_dir + country_low)
admin_low = str(self.admin).lower()
if os.path.exists(admin_low):
pass
else:
os.mkdir(admin_low)
return "Project created......\n"
def create_template(self, dir_template):
os.chdir(dir_template)
tabella = dbf.Table('monthly_values', 'month C(10);mean N(5,1)')
return tabella.filename