def download_and_unzip_post(config,
rootpath,
hot_run=True,
disable_progress=False):
"""
download_and_unzip_post(config, rootpath, dest_path, hot_run=True, disable_progress=False)
Function to download the data by category from a post request.
Inputs
------
config : Dict
Configuration data for the category to download
rootpath : str
Absolute path of the repository
hot_run : Bool (default True)
When true the data are downloaded
When false, the workflow is run without downloading and unzipping
disable_progress : Bool (default False)
When true the progress bar to download data is disabled
Outputs
-------
True when download is successful, False otherwise
"""
resource = config["category"]
# load data for post method
postdata = config["urls"]["post"]
# remove url feature
url = postdata.pop("url")
file_path = os.path.join(config["destination"], os.path.basename(url))
if hot_run:
if os.path.exists(file_path):
os.remove(file_path)
# try:
logger.info(f"Downloading resource '{resource}' from cloud '{url}'.")
progress_retrieve(url,
file_path,
data=postdata,
disable_progress=disable_progress)
# if the file is a zipfile and unzip is enabled
# then unzip it and remove the original file
if config.get("unzip", False):
with ZipFile(file_path, "r") as zipfile:
zipfile.extractall(config["destination"])
os.remove(file_path)
logger.info(f"Downloaded resource '{resource}' from cloud '{url}'.")
# except:
# logger.warning(f"Failed download resource '{resource}' from cloud '{url}'.")
# return False
return True
def download_and_unzip_zenodo(config,
rootpath,
hot_run=True,
disable_progress=False):
"""
download_and_unzip_zenodo(config, rootpath, dest_path, hot_run=True, disable_progress=False)
Function to download and unzip the data from zenodo
Inputs
------
config : Dict
Configuration data for the category to download
rootpath : str
Absolute path of the repository
hot_run : Bool (default True)
When true the data are downloaded
When false, the workflow is run without downloading and unzipping
disable_progress : Bool (default False)
When true the progress bar to download data is disabled
Outputs
-------
True when download is successful, False otherwise
"""
resource = config["category"]
file_path = os.path.join(rootpath, "tempfile.zip")
url = config["urls"]["zenodo"]
if hot_run:
try:
logger.info(
f"Downloading resource '{resource}' from cloud '{url}'")
progress_retrieve(url,
file_path,
disable_progress=disable_progress)
logger.info(f"Extracting resources")
with ZipFile(file_path, "r") as zipObj:
# Extract all the contents of zip file in current directory
zipObj.extractall(path=config["destination"])
os.remove(file_path)
logger.info(
f"Downloaded resource '{resource}' from cloud '{url}'.")
except:
logger.warning(
f"Failed download resource '{resource}' from cloud '{url}'.")
return False
return True
:ref:`toplevel_cf`
**Outputs**
- ``cutouts/{cutout}``: weather data from either the `ERA5 <https://www.ecmwf.int/en/forecasts/datasets/reanalysis-datasets/era5>`_ reanalysis weather dataset or `SARAH-2 <https://wui.cmsaf.eu/safira/action/viewProduktSearch>`_ satellite-based historic weather data.
.. seealso::
For details see :mod:`build_cutout` and read the `atlite documentation <https://atlite.readthedocs.io>`_.
"""
import logging, os, tarfile
from _helpers import progress_retrieve
logger = logging.getLogger(__name__)
if __name__ == "__main__":
if snakemake.config['tutorial']:
url = "https://zenodo.org/record/3518020/files/pypsa-eur-tutorial-cutouts.tar.xz"
else:
url = "https://zenodo.org/record/3517949/files/pypsa-eur-cutouts.tar.xz"
tarball_fn = "./cutouts.tar.xz"
progress_retrieve(url, tarball_fn)
tarfile.open(tarball_fn).extractall()
os.remove(tarball_fn)
**Outputs**
- ``resources/natura.tiff``: Rasterized version of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas to reduce computation times.
.. seealso::
For details see :mod:`build_natura_raster`.
"""
import logging
logger = logging.getLogger(__name__)
from _helpers import progress_retrieve, configure_logging
if __name__ == "__main__":
if 'snakemake' not in globals():
from _helpers import mock_snakemake
snakemake = mock_snakemake('retrieve_natura_raster')
configure_logging(
snakemake
) # TODO Make logging compatible with progressbar (see PR #102)
url = "https://zenodo.org/record/3518215/files/natura.tiff"
logger.info(f"Downloading natura raster from '{url}'.")
progress_retrieve(url, snakemake.output[0])
logger.info(f"Natura raster available as '{snakemake.output[0]}'.")
def download_and_unzip_protectedplanet(config,
rootpath,
hot_run=True,
disable_progress=False):
"""
download_and_unzip_protectedplanet(config, rootpath, dest_path, hot_run=True, disable_progress=False)
Function to download and unzip the data by category from protectedplanet
Inputs
------
config : Dict
Configuration data for the category to download
rootpath : str
Absolute path of the repository
hot_run : Bool (default True)
When true the data are downloaded
When false, the workflow is run without downloading and unzipping
disable_progress : Bool (default False)
When true the progress bar to download data is disabled
Outputs
-------
True when download is successful, False otherwise
"""
resource = config["category"]
file_path = os.path.join(rootpath, "tempfile_wpda.zip")
url = config["urls"]["protectedplanet"]
if hot_run:
if os.path.exists(file_path):
os.remove(file_path)
try:
logger.info(
f"Downloading resource '{resource}' from cloud '{url}'.")
progress_retrieve(url,
file_path,
disable_progress=disable_progress)
zip_obj = ZipFile(file_path, "r")
# list of zip files, which contains the shape files
zip_files = [
fname for fname in zip_obj.namelist() if fname.endswith(".zip")
]
# extract the nested zip files
for fzip in zip_files:
# final path of the file
inner_zipname = os.path.join(config["destination"], fzip)
zip_obj.extract(fzip, path=config["destination"])
with ZipFile(inner_zipname, "r") as nested_zip:
nested_zip.extractall(path=config["destination"])
# remove inner zip file
os.remove(inner_zipname)
# remove outer zip file
os.remove(file_path)
logger.info(
f"Downloaded resource '{resource}' from cloud '{url}'.")
except:
logger.warning(
f"Failed download resource '{resource}' from cloud '{url}'.")
return False
return True
enable:
build_natura_raster:
.. seealso::
Documentation of the configuration file ``config.yaml`` at
:ref:`toplevel_cf`
**Outputs**
- ``resources/natura.tiff``: Rasterized version of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas to reduce computation times.
.. seealso::
For details see :mod:`build_natura_raster`.
"""
import logging, os
from _helpers import progress_retrieve
logger = logging.getLogger(__name__)
if __name__ == "__main__":
d = './resources'
if not os.path.exists(d):
os.makedirs(d)
progress_retrieve("https://zenodo.org/record/3518215/files/natura.tiff",
"resources/natura.tiff")