def __download_climate_observations_data(remote_file: str) -> bytes:
try:
zip_file = download_file_from_dwd(remote_file)
except InvalidURL as e:
raise InvalidURL(
f"Error: the station data {remote_file} could not be reached."
) from e
except Exception:
raise FailedDownload(f"Download failed for {remote_file}")
try:
zip_file_opened = ZipFile(zip_file)
# Files of archive
archive_files = zip_file_opened.namelist()
for file in archive_files:
# If found file load file in bytes, close zipfile and return bytes
if file.startswith(PRODUCT_FILE_IDENTIFIER):
file_in_bytes = zip_file_opened.open(file).read()
zip_file_opened.close()
return file_in_bytes
# If whatsoever no file was found and returned already throw exception
raise ProductFileNotFound(
f"The archive of {remote_file} does not hold a 'produkt' file.")
except BadZipFile as e:
raise BadZipFile(
f"The archive of {remote_file} seems to be corrupted.") from e
def _create_meta_index_for_climate_observations(
parameter: Parameter, time_resolution: TimeResolution,
period_type: PeriodType) -> pd.DataFrame:
"""Function used to create meta index DataFrame parsed from the text files that are
located in each data section of the station data directory of the weather service.
Args:
parameter: observation measure
time_resolution: frequency/granularity of measurement interval
period_type: current, recent or historical files
Return:
DataFrame with parsed columns of the corresponding text file. Columns are
translated into English and data is not yet complete as file existence is
not checked.
"""
parameter_path = build_path_to_parameter(parameter, time_resolution,
period_type)
url = reduce(
urljoin,
[
DWD_SERVER,
DWD_CDC_PATH,
DWDCDCBase.CLIMATE_OBSERVATIONS.value,
parameter_path,
],
)
files_server = list_remote_files(url, recursive=True)
# Find the one meta file from the files listed on the server
meta_file = _find_meta_file(files_server, url)
try:
file = download_file_from_dwd(meta_file)
except InvalidURL as e:
raise InvalidURL(
f"Error: reading metadata {meta_file} file failed.") from e
meta_index = pd.read_fwf(
filepath_or_buffer=file,
colspecs=METADATA_FIXED_COLUMN_WIDTH,
skiprows=[1],
dtype=str,
encoding="ISO-8859-1",
)
# Fix column names, as header is not aligned to fixed column widths
meta_index.columns = "".join([
column for column in meta_index.columns
if "unnamed" not in column.lower()
]).split(" ")
meta_index = meta_index.rename(columns=str.upper)
meta_index = meta_index.rename(columns=GERMAN_TO_ENGLISH_COLUMNS_MAPPING)
return meta_index.astype(METADATA_DTYPE_MAPPING)
def _download_radolan_data(remote_radolan_filepath: str) -> BytesIO:
"""
Function (cached) that downloads the RADOLAN file
Args:
remote_radolan_filepath: the file path to the file on the DWD server
Returns:
the file in binary, either an archive of one file or an archive of multiple
files
"""
return download_file_from_dwd(remote_radolan_filepath)
def _download_metadata_file_for_1minute_precipitation(metadata_file: str) -> BytesIO:
"""A function that simply opens a filepath with help of the urllib library and then
writes the content to a BytesIO object and returns this object. For this case as it
opens lots of requests (there are approx 1000 different files to open for
1minute data), it will do the same at most three times for one file to assure
success reading the file.
Args:
metadata_file (str) - the file that shall be downloaded and returned as bytes.
Return:
A BytesIO object to which the opened file was written beforehand.
"""
try:
file = download_file_from_dwd(metadata_file)
except InvalidURL as e:
raise InvalidURL(f"Reading metadata {metadata_file} file failed.") from e
return file
def _download_generic_data_cached(url: str) -> Tuple[str, BytesIO]:
return url, download_file_from_dwd(url)