def get(
datasets: dict = GISCO_DATASETS,
crs: CRS = CRS.from_epsg(3035)
) -> gpd.GeoDataFrame:
"""
Retrieve NUTS, LAU and countries from GISCO API and make a single, consistent GDF.
Parameters
----------
datasets : dict
dict with API URLs.
crs : str, optional
Spatial Reference System. The default is "EPSG:4326".
Returns
-------
admin_units : GeoDataFrame
Table with all administrative units.
"""
source_crs_code = crs.to_epsg()
logging.info("Downloading countries...")
countries = gpd.read_file(datasets["countries"].format(source_crs_code),
crs=crs.to_string())
logging.info("Downloading NUTS...")
nuts = gpd.read_file(datasets["nuts"].format(source_crs_code),
crs=crs.to_string())
logging.info("Downloading LAU...")
lau = gpd.read_file(datasets["lau"].format(source_crs_code),
crs=crs.to_string())
logging.info("Done.")
# Convert to lower case
countries.columns = countries.columns.str.lower()
nuts.columns = nuts.columns.str.lower()
lau.columns = lau.columns.str.lower()
# Create consistent columns across ds
lau = lau.rename({"lau_name": "name"}, axis=1)
lau["levl_code"] = 4
nuts = nuts.rename({"name_latn": "name"}, axis=1)
countries = countries.rename({"cntr_name": "name"}, axis=1)
countries = countries.rename({"cntr_id": "cntr_code"}, axis=1)
countries["levl_code"] = 0
# EU+ countries are included both in NUTS (level 0) and in "countries"
# Discard then NUTS level 0
nuts_noEU = nuts.loc[~nuts.id.isin(countries.id), :]
admin_units = pd.concat([countries, nuts_noEU, lau],
axis=0,
ignore_index=True)
admin_units = gpd.GeoDataFrame(
admin_units.
loc[:,
["fid", "name", "name_engl", "cntr_code", "levl_code", "geometry"]]
)
# New level codes
admin_units.levl_code = admin_units.levl_code.replace({
0: "country",
1: "NUTS1",
2: "NUTS2",
3: "NUTS3",
4: "LAU"
})
# Convert to ISO 3166-1 alpha-2
transl = {"UK": "GB", "EL": "GR"}
admin_units["fid"] = admin_units["fid"].replace(transl)
admin_units["cntr_code"] = admin_units["cntr_code"].replace(transl)
admin_units.crs = crs.to_string()
return admin_units
def test_to_string__no_auth():
proj = CRS("+proj=latlong +ellps=GRS80 +towgs84=-199.87,74.79,246.62")
assert (proj.to_string(
) == "+proj=latlong +ellps=GRS80 +towgs84=-199.87,74.79,246.62 +type=crs")
def prepareRaster(
df: pd.DataFrame,
crs: CRS = CRS.from_epsg(3035),
variable: str = "",
delete_orig: bool = False,
):
"""
Convert original raster or NetCDF into EnerMaps rasters (single band, GeoTiff, EPSG:3035).
Parameters
----------
df : DataFrame.
Results of API extraction.
crs : pyproj.crs.CRS.
coordinate reference system.
variable : str, optional.
Variable of NETCDF.
delete_orig : bool, optional.
Set to True to delete original downloaded file (e.g. NetCDF).
Returns
-------
df : DataFrame
Results with schema for EnerMaps data table
"""
dicts = []
for i, row in df.iterrows():
filename = row["value"]
if filename.startswith("http"):
filename = "/vsicurl/" + filename
if filename[-2:] == "nc":
src_ds = gdal.Open("NETCDF:{0}:{1}".format(filename, variable))
else:
src_ds = gdal.Open(filename)
# Override function parameter
if "variable" in row.index:
variable = row["variable"]
if "crs" in df.columns:
source_wkt = osr.SpatialReference()
source_wkt.ImportFromEPSG(row.crs.to_epsg())
source_wkt = source_wkt.ExportToPrettyWkt()
source_crs = CRS.from_wkt(source_wkt)
else:
prj = src_ds.GetProjection()
srs = osr.SpatialReference(wkt=prj)
source_crs = CRS.from_epsg(srs.GetAttrValue("authority", 1))
dest_wkt = osr.SpatialReference()
dest_wkt.ImportFromEPSG(crs.to_epsg())
dest_wkt = dest_wkt.ExportToPrettyWkt()
for b in range(src_ds.RasterCount):
my_dict = {}
b += 1
dest_filename = Path(filename).stem
dest_filename += "_band" + str(b)
# Translating to make sure that the raster settings are consistent for each band
logging.info("Translating band {}".format(b))
os.system(
"gdal_translate {filename} {dest_filename}.tif -b {b} -of GTIFF --config GDAL_PAM_ENABLED NO -co COMPRESS=DEFLATE -co BIGTIFF=YES".format(
filename=filename, dest_filename=dest_filename, b=b
)
)
# Reprojecting if needed
if source_crs.to_epsg() != crs.to_epsg():
logging.info(
"Warping from {} to {}".format(source_crs.to_epsg(), crs.to_epsg())
)
intermediate_filename = dest_filename + ".tif" # from previous step
dest_filename += "_{}".format(crs.to_epsg())
os.system(
"gdalwarp {intermediate_filename} {dest_filename}.tif -of GTIFF -s_srs {sourceSRS} -t_srs {outputSRS} --config GDAL_PAM_ENABLED NO -co COMPRESS=DEFLATE -co BIGTIFF=YES".format(
intermediate_filename=intermediate_filename,
dest_filename=dest_filename,
outputSRS=crs.to_string(),
sourceSRS=source_crs.to_string(),
)
)
os.remove(intermediate_filename)
dest_filename += ".tif"
logging.info(dest_filename)
my_dict["start_at"] = row["start_at"] + pd.Timedelta(hours=row["dt"]) * (
b - 1
)
my_dict["z"] = row["z"]
my_dict["dt"] = row["dt"]
my_dict["unit"] = row["unit"]
my_dict["variable"] = variable
my_dict["fid"] = dest_filename
my_dict["israster"] = True
dicts.append(my_dict)
data = pd.DataFrame(
dicts,
columns=[
"start_at",
"fields",
"variable",
"value",
"ds_id",
"fid",
"dt",
"z",
"unit",
"israster",
],
)
if delete_orig:
os.remove(filename)
return data
