def get_kba_table():
"""
Get the "kfz" table for all vehicles and the "pkw" table for more
statistics about passenger cars.
Returns
-------
namedtuple
Examples
--------
>>> table = get_kba_table()
>>> kfz = table.kfz
>>> print(type(kfz))
<class 'pandas.core.frame.DataFrame'>
"""
kba_table = namedtuple("kba_table", "kfz pkw")
kba_filename = os.path.join(cfg.get("paths", "general"),
cfg.get("mobility", "table_kba"))
# Download table if it does not exit
if not os.path.isfile(kba_filename):
tools.download_file(kba_filename, cfg.get("mobility", "url_kba"))
return kba_table(
kfz=format_kba_table(kba_filename, "Kfz_u_Kfz_Anh"),
pkw=format_kba_table(kba_filename, "Pkw"),
)
def test_01_download_reegis_power_plants():
url = "https://osf.io/ude5c/download"
path = cfg.get("paths", "powerplants")
file = "reegis_pp_test.h5"
filename = os.path.join(path, file)
download_file(filename, url)
def setup_func():
"""Download pp-file from osf."""
url = "https://osf.io/m435r/download"
path = cfg.get("paths", "demand")
file = "heat_profile_state_2014_weather_2014.csv"
filename = os.path.join(path, file)
download_file(filename, url)
url = "https://osf.io/6vmdh/download"
file = "oep_ego_demand_combined.h5"
filename = os.path.join(path, file)
download_file(filename, url)
def get_ew_shp_file(year):
if year < 2011:
logging.error("Shapefile with inhabitants are available since 2011.")
logging.error("Try to find another source to get older data sets.")
raise AttributeError('Years < 2011 are not allowed in this function.')
outshp = os.path.join(cfg.get('paths', 'inhabitants'),
'VG250_VWG_' + str(year) + '.shp')
if not os.path.isfile(outshp):
url = cfg.get('inhabitants', 'url_geodata_ew').format(year=year,
var1='{0}')
filename_zip = os.path.join(cfg.get('paths', 'inhabitants'),
cfg.get('inhabitants', 'vg250_ew_zip'))
msg = tools.download_file(filename_zip, url.format('ebene'))
if msg == 404:
logging.warning("Wrong URL. Try again with different URL.")
tools.download_file(filename_zip,
url.format('ebenen'),
overwrite=True)
zip_ref = zipfile.ZipFile(filename_zip, 'r')
zip_ref.extractall(cfg.get('paths', 'inhabitants'))
zip_ref.close()
subs = next(os.walk(cfg.get('paths', 'inhabitants')))[1]
mysub = None
for sub in subs:
if 'vg250' in sub:
mysub = sub
pattern_path = list()
pattern_path.append(
os.path.join(cfg.get('paths', 'inhabitants'), mysub,
'vg250-ew_ebenen', 'VG250_VWG*'))
pattern_path.append(
os.path.join(cfg.get('paths', 'inhabitants'), mysub,
'vg250-ew_ebenen', 'vg250_vwg*'))
pattern_path.append(
os.path.join(cfg.get('paths', 'inhabitants'), mysub,
'vg250_ebenen-historisch',
'de{0}12'.format(str(year)[-2:]), 'vg250_vwg*'))
for pa_path in pattern_path:
for file in glob.glob(pa_path):
file_new = os.path.join(cfg.get('paths', 'inhabitants'),
'VG250_VWG_' + str(year) + file[-4:])
shutil.copyfile(file, file_new)
shutil.rmtree(os.path.join(cfg.get('paths', 'inhabitants'), mysub))
os.remove(filename_zip)
def get_mileage_table():
"""
Download mileage table from the KBA (Kraftfahrtbundesamt) and store it
locally.
"""
url = (
"https://www.kba.de/SharedDocs/Publikationen/DE/Statistik/"
"Kraftverkehr/VK/2018/vk_2018_xlsx.xlsx?__blob=publicationFile&v=22")
mileage_filename = os.path.join(cfg.get("paths", "general"),
"mileage_table_kba.xlsx")
# Download table if it does not exit
if not os.path.isfile(mileage_filename):
tools.download_file(mileage_filename, url)
return mileage_filename
def get_bmwi_energiedaten_file(overwrite=False):
filename = os.path.join(cfg.get('paths', 'general'),
cfg.get('bmwi', 'energiedaten'))
logging.debug("Return status from energiedaten file: {0}".format(
tools.download_file(filename,
cfg.get('bmwi', 'url_energiedaten'),
overwrite=overwrite)))
return filename
def show_download_image(name, file_types):
create_subplot((12, 4.4))
fn = os.path.join(cfg.get("paths", "figure_source"),
"{0}.png".format(name))
img = mpimg.imread(fn)
plt.imshow(img)
plt.axis("off")
plt.subplots_adjust(left=0, top=0.93, bottom=0, right=1)
url = ("https://raw.githubusercontent.com/uvchik/reegis_phd/master/"
"reegis_phd/data/figures/{0}.{1}")
fn = os.path.join(cfg.get("paths", "figures"), "{0}.{1}")
for suffix in file_types:
download_file(fn.format(name, suffix), url.format(name, suffix), True)
plt.title("Image source downloaded to: {0}".format(
fn.format(name, file_types[0])))
return "{0}.png".format(name), None
def get_bmwi_energiedaten_file(overwrite=False):
"""Download BMWi energy data table."""
filename = os.path.join(cfg.get("paths", "general"),
cfg.get("bmwi", "energiedaten"))
logging.debug("Return status from energiedaten file: {0}".format(
tools.download_file(
filename,
cfg.get("bmwi", "url_energiedaten"),
overwrite=overwrite,
)))
return filename
def setup_func():
"""Download pp-file from osf."""
downloads = [
("n7ahr", "geothermal"),
("5n7t3", "hydro"),
("2qwv7", "solar"),
("9dvpf", "wind"),
]
for d in downloads:
url = "https://osf.io/{0}/download".format(d[0])
path = os.path.join(cfg.get("paths", "feedin"), "de21", "2014")
file = "2014_feedin_de21_normalised_{0}.csv".format(d[1])
filename = os.path.join(path, file)
os.makedirs(path, exist_ok=True)
download_file(filename, url)
src = os.path.join(os.path.dirname(__file__), "data", "windzone_de21.csv")
trg = os.path.join(cfg.get("paths", "powerplants"), "windzone_de21.csv")
copyfile(src, trg)
def download_coastdat_data(filename=None,
year=None,
url=None,
test_only=False,
overwrite=True):
"""
Download coastdat data set from internet source.
Parameters
----------
filename : str
Full path with the filename, where the downloaded file will be stored.
year : int or None
Year of the weather data set. If a url is passed this value will be
ignored because it is used to create the default url.
url : str or None
Own url can be used if the default url does not work an one found an
alternative valid url.
test_only : bool
If True the the url is tested but the file will not be downloaded
(default: False).
overwrite : bool
If True the file will be downloaded even if it already exist.
(default: True)
Returns
-------
str or None : If the url is valid the filename is returned otherwise None.
Examples
--------
>>> download_coastdat_data(year=2014, test_only=True)
'coastDat2_de_2014.h5'
>>> print(download_coastdat_data(url='https://osf.io/url', test_only=True))
None
>>> download_coastdat_data(filename='w14.hd5', year=2014) # doctest: +SKIP
"""
if url is None:
url_ids = cfg.get_dict("coastdat_url_id")
url_id = url_ids.get(str(year), None)
if url_id is not None:
url = cfg.get("coastdat", "basic_url").format(url_id=url_id)
if url is not None and not test_only:
response = requests.get(url, stream=True)
if response.status_code == 200:
msg = "Downloading the coastdat2 file of {0} from {1} ..."
logging.info(msg.format(year, url))
if filename is None:
headers = response.headers["Content-Disposition"]
filename = (headers.split("; ")[1].split("=")[1].replace(
'"', ""))
tools.download_file(filename, url, overwrite=overwrite)
return filename
else:
raise ValueError("URL not valid: {0}".format(url))
elif url is not None and test_only:
response = requests.get(url, stream=True)
if response.status_code == 200:
headers = response.headers["Content-Disposition"]
filename = headers.split("; ")[1].split("=")[1].replace('"', "")
else:
filename = None
return filename
else:
raise ValueError("No URL found for {0}".format(year))
def get_ew_shp_file(year):
"""
Parameters
----------
year
Returns
-------
Examples
--------
>>> print(get_ew_shp_file(2014)[-35:])
data/inhabitants/VG250_VWG_2014.shp
"""
if year < 2011:
logging.error("Shapefile with inhabitants are available since 2011.")
logging.error("Try to find another source to get older data sets.")
raise AttributeError("Years < 2011 are not allowed in this function.")
outshp = os.path.join(cfg.get("paths", "inhabitants"),
"VG250_VWG_" + str(year) + ".shp")
if not os.path.isfile(outshp):
url = cfg.get("inhabitants", "url_geodata_ew").format(year=year,
var1="{0}")
filename_zip = os.path.join(
cfg.get("paths", "inhabitants"),
cfg.get("inhabitants", "vg250_ew_zip"),
)
msg = tools.download_file(filename_zip, url.format("ebene"))
if msg == 404:
logging.warning("Wrong URL. Try again with different URL.")
tools.download_file(filename_zip,
url.format("ebenen"),
overwrite=True)
zip_ref = zipfile.ZipFile(filename_zip)
zip_ref.extractall(cfg.get("paths", "inhabitants"))
zip_ref.close()
subs = next(os.walk(cfg.get("paths", "inhabitants")))[1]
mysub = None
for sub in subs:
if "vg250" in sub:
mysub = sub
pattern_path = list()
pattern_path.append(
os.path.join(
cfg.get("paths", "inhabitants"),
mysub,
"vg250-ew_ebenen",
"VG250_VWG*",
))
pattern_path.append(
os.path.join(
cfg.get("paths", "inhabitants"),
mysub,
"vg250-ew_ebenen",
"vg250_vwg*",
))
pattern_path.append(
os.path.join(
cfg.get("paths", "inhabitants"),
mysub,
"vg250_ebenen-historisch",
"de{0}12".format(str(year)[-2:]),
"vg250_vwg*",
))
for pa_path in pattern_path:
for file in glob.glob(pa_path):
file_new = os.path.join(
cfg.get("paths", "inhabitants"),
"VG250_VWG_" + str(year) + file[-4:],
)
shutil.copyfile(file, file_new)
shutil.rmtree(os.path.join(cfg.get("paths", "inhabitants"), mysub))
os.remove(filename_zip)
return outshp
def get_ewi_data():
"""
Returns
-------
namedtuple
TODO: Keep this in deflex???
Examples
--------
# >>> ewi_data = get_ewi_data()
# >>> round(ewi_data.fuel_costs.loc["hard coal", "value"], 2)
# 11.28
"""
# Download file
url = ("https://www.ewi.uni-koeln.de/cms/wp-content/uploads/2019/12"
"/EWI_Merit_Order_Tool_2019_1_4.xlsm")
fn = os.path.join(cfg.get("paths", "general"), "ewi.xls")
tools.download_file(fn, url)
# Create named tuple with all sub tables
ewi_tables = {
"fuel_costs": {
"skiprows": 7,
"usecols": "C:F",
"nrows": 7
},
"transport_costs": {
"skiprows": 21,
"usecols": "C:F",
"nrows": 7
},
"variable_costs": {
"skiprows": 31,
"usecols": "C:F",
"nrows": 8
},
"downtime_factor": {
"skiprows": 31,
"usecols": "H:K",
"nrows": 8,
"scale": 0.01,
},
"emission": {
"skiprows": 31,
"usecols": "M:P",
"nrows": 7
},
"co2_price": {
"skiprows": 17,
"usecols": "C:F",
"nrows": 1
},
}
ewi_data = {}
cols = ["fuel", "value", "unit", "source"]
xls = pd.ExcelFile(fn)
for table in ewi_tables.keys():
tmp = xls.parse("Start", header=[0],
**ewi_tables[table]).replace(TRANSLATION_FUEL)
tmp.drop_duplicates(tmp.columns[0], keep="first", inplace=True)
tmp.columns = cols
ewi_data[table] = tmp.set_index("fuel")
if "scale" in ewi_tables[table]:
ewi_data[table]["value"] *= ewi_tables[table]["scale"]
return SimpleNamespace(**ewi_data)
def get_ego_data(osf=True, sectors=False, query="?where=version=v0.4.5"):
"""
Parameters
----------
osf : bool
If True the file will be downloaded from the osf page instead of of
selected from the database. You may not get the latest version.
(default: False)
sectors : bool
By default (False) only the total comsumption is returned. If "True"
the consumption devided by sectors will be returned.
query : str
Database query to filter the data set.
(default: '?where=version=v0.4.5')
Returns
-------
Examples
--------
>>> from reegis import openego
>>> # download from file (faster)
>>> openego.get_ego_data() # doctest: +SKIP
>>> # download from oedb database (get latest updates, very slow)
>>> openego.get_ego_data(osf=False) # doctest: +SKIP
"""
oep_url = "http://oep.iks.cs.ovgu.de/api/v0"
local_path = cfg.get("paths", "ego")
fn_large_consumer = os.path.join(
local_path, cfg.get("open_ego", "ego_large_consumers"))
fn_load_areas = os.path.join(local_path,
cfg.get("open_ego", "ego_load_areas"))
# Large scale consumer
schema = "model_draft"
table = "ego_demand_hv_largescaleconsumer"
query_lsc = ""
download_oedb(oep_url, schema, table, query_lsc, fn_large_consumer)
large_consumer = pd.read_csv(fn_large_consumer, index_col=[0])
msg = ("\nYou are going to download the load areas from file created "
"2019-10-09.\nThis is much faster and useful for most users but you"
" may find more actual data on the oedb database.\n"
"Please check: https://openenergy-platform.org/dataedit/view/demand"
"/ego_dp_loadarea \n"
"Use 'openego.get_ego_data(osf=False)' to fetch data from oedb.\n")
# Load areas
if osf is True:
warnings.warn(msg)
url = cfg.get("open_ego", "osf_url")
tools.download_file(fn_load_areas, url)
load_areas = pd.DataFrame(pd.read_csv(fn_load_areas, index_col=[0]))
else:
schema = "demand"
table = "ego_dp_loadarea"
download_oedb(oep_url, schema, table, query, fn_load_areas)
load_areas = pd.DataFrame(pd.read_csv(fn_load_areas, index_col=[0]))
load_areas.rename(columns={"sector_consumption_sum": "consumption"},
inplace=True)
if sectors:
large_consumer["sector_consumption_large_consumers"] = large_consumer[
"consumption"]
cols_lc = [c for c in large_consumer.columns if "consumption" in c
] + ["geom_centre"]
cols_la = [c for c in load_areas.columns if "consumption" in c
] + ["geom_centre"]
else:
cols_la = ["consumption", "geom_centre"]
cols_lc = ["consumption", "geom_centre"]
load = pd.concat([load_areas[cols_la], large_consumer[cols_lc]])
load = load.rename(columns={"geom_centre": "geom"})
return load.reset_index(drop=True)