def test_collect_dwd_data_success():
""" Test for data collection """
"""
1. Scenario
This scenario makes sure we take fresh data and write it to the given folder, thus
we can run just another test afterwards as no old data is used
"""
assert collect_climate_observations_data(
station_ids=[1],
parameter=Parameter.CLIMATE_SUMMARY,
time_resolution=TimeResolution.DAILY,
period_type=PeriodType.HISTORICAL,
prefer_local=False,
write_file=True,
tidy_data=False,
).equals(TEST_FILE)
"""
2. Scenario
This scenario tries to get the data from the given folder. This data was placed by
the first test and is now restored
"""
assert collect_climate_observations_data(
station_ids=[1],
parameter=Parameter.CLIMATE_SUMMARY,
time_resolution=TimeResolution.DAILY,
period_type=PeriodType.HISTORICAL,
prefer_local=True,
write_file=True,
tidy_data=False,
).equals(TEST_FILE)
def test_collect_daily_vanilla():
""" Test for data collection with real data """
data = collect_climate_observations_data(
station_id=1048,
parameter_set=DWDObservationParameterSet.CLIMATE_SUMMARY,
resolution=DWDObservationResolution.DAILY,
period=DWDObservationPeriod.RECENT,
)
assert list(data.columns.values) == [
"STATION_ID",
"DATE",
"QN_3",
"FX",
"FM",
"QN_4",
"RSK",
"RSKF",
"SDK",
"SHK_TAG",
"NM",
"VPM",
"PM",
"TMK",
"UPM",
"TXK",
"TNK",
"TGK",
]
def test_collect_daily_vanilla():
""" Test for data collection with real data """
data = collect_climate_observations_data(
station_ids=[1048],
parameter=Parameter.CLIMATE_SUMMARY,
time_resolution=TimeResolution.DAILY,
period_type=PeriodType.RECENT,
tidy_data=False,
)
assert list(data.columns.values) == [
"STATION_ID",
"DATE",
"QN_3",
"FX",
"FM",
"QN_4",
"RSK",
"RSKF",
"SDK",
"SHK_TAG",
"NM",
"VPM",
"PM",
"TMK",
"UPM",
"TXK",
"TNK",
"TGK",
]
def test_collect_dwd_data_empty():
""" Test for data collection with no available data """
assert collect_climate_observations_data(
station_id=1048,
parameter_set=DWDObservationParameterSet.CLIMATE_SUMMARY,
resolution=DWDObservationResolution.DAILY,
period=DWDObservationPeriod.RECENT,
).empty
def test_collect_hourly_vanilla():
""" Test for data collection with real data """
data = collect_climate_observations_data(
station_id=1048,
parameter_set=DWDObservationParameterSet.TEMPERATURE_AIR,
resolution=DWDObservationResolution.HOURLY,
period=DWDObservationPeriod.RECENT,
)
assert list(data.columns.values) == [
"STATION_ID",
"DATE",
"QN_9",
"TT_TU",
"RF_TU",
]
def test_collect_dwd_data_empty():
""" Test for data collection with no available data """
"""
1. Scenario
Test for request where no data is available
"""
assert collect_climate_observations_data(
station_ids=[1048],
parameter=Parameter.CLIMATE_SUMMARY,
time_resolution=TimeResolution.DAILY,
period_type=PeriodType.RECENT,
prefer_local=True,
write_file=False,
tidy_data=False,
).empty
def test_collect_hourly_vanilla():
""" Test for data collection with real data """
data = collect_climate_observations_data(
station_ids=[1048],
parameter=Parameter.TEMPERATURE_AIR,
time_resolution=TimeResolution.HOURLY,
period_type=PeriodType.RECENT,
tidy_data=False,
)
assert list(data.columns.values) == [
"STATION_ID",
"DATE",
"QN_9",
"TT_TU",
"RF_TU",
]
def collect_data(self) -> Generator[pd.DataFrame, None, None]:
"""
Method to collect data for a defined request. The function is build as generator
in order to not cloak the memory thus if the user wants the data as one pandas
DataFrame the generator has to be casted to a DataFrame manually via
pd.concat(list(request.collect_data()).
:return: A generator yielding a pandas.DataFrame per station.
"""
for station_id in self.station_ids:
df_station = pd.DataFrame()
for parameter in self.parameter:
df_parameter_period = pd.DataFrame()
for period_type in self.period_type:
try:
df_period = collect_climate_observations_data(
station_ids=[station_id],
parameter=parameter,
time_resolution=self.time_resolution,
period_type=period_type,
folder=self.folder,
prefer_local=self.prefer_local,
write_file=self.write_file,
tidy_data=self.tidy_data,
humanize_column_names=self.humanize_column_names,
)
except InvalidParameterCombination:
log.info(
f"Combination for "
f"{parameter.value}/"
f"{self.time_resolution.value}/"
f"{period_type} does not exist and is skipped.")
continue
# Filter out values which already are in the DataFrame
try:
df_period = df_period[
~df_period[DWDMetaColumns.DATE.value].
isin(df_parameter_period[DWDMetaColumns.DATE.value]
)]
except KeyError:
pass
df_parameter_period = df_parameter_period.append(
df_period, ignore_index=True)
df_station = df_station.append(df_parameter_period,
ignore_index=True)
# Filter for dates range if start_date and end_date are defined
if self.start_date:
df_station = df_station[
(df_station[DWDMetaColumns.DATE.value] >= self.start_date)
& (df_station[DWDMetaColumns.DATE.value] <= self.end_date)]
# Empty dataframe should be skipped
if df_station.empty:
continue
yield df_station
def _collect_data(
self, station_id: int, parameter_set: DWDObservationParameterSet
) -> pd.DataFrame:
"""
Method to collect data for one specified parameter. Manages restoring,
collection and storing of data, transformation and combination of different
periods.
Args:
station_id: station id for which parameter is collected
parameter_set: chosen parameter that is collected
Returns:
pandas.DataFrame for given parameter of station
"""
df_parameter = pd.DataFrame()
for period_type in self.periods:
parameter_identifier = build_parameter_set_identifier(
parameter_set, self.resolution, period_type, station_id
)
storage = None
if self.storage:
storage = self.storage.hdf5(
parameter=parameter_set,
resolution=self.resolution,
period=period_type,
)
df_period = storage.restore(station_id)
if not df_period.empty:
df_parameter = df_parameter.append(df_period)
continue
log.info(f"Acquiring observations data for {parameter_identifier}.")
try:
df_period = collect_climate_observations_data(
station_id, parameter_set, self.resolution, period_type
)
except InvalidParameterCombination:
log.info(
f"Invalid combination {parameter_set.value}/"
f"{self.resolution.value}/{period_type} is skipped."
)
df_period = pd.DataFrame()
if self.storage and self.storage.persist:
storage.store(station_id=station_id, df=df_period)
# Filter out values which already are in the DataFrame
try:
df_period = df_period[
~df_period[DWDMetaColumns.DATE.value].isin(
df_parameter[DWDMetaColumns.DATE.value]
)
]
except KeyError:
pass
df_parameter = df_parameter.append(df_period)
if self.tidy_data:
df_parameter = df_parameter.dwd.tidy_up_data()
df_parameter.insert(2, DWDMetaColumns.PARAMETER.value, parameter_set.name)
# Assign meaningful column names (humanized).
if self.humanize_column_names:
hcnm = self._create_humanized_column_names_mapping(
self.resolution, parameter_set
)
if self.tidy_data:
df_parameter[DWDMetaColumns.ELEMENT.value] = df_parameter[
DWDMetaColumns.ELEMENT.value
].apply(lambda x: hcnm[x])
else:
df_parameter = df_parameter.rename(columns=hcnm)
return df_parameter
def _collect_station_parameter(
self,
station_id: str,
parameter: Tuple[Union[DWDObservationParameter,
DWDObservationParameterSet],
DWDObservationParameterSet, ],
) -> pd.DataFrame:
"""
Method to collect data for one specified parameter. Manages restoring,
collection and storing of data, transformation and combination of different
periods.
Args:
station_id: station id for which parameter is collected
parameter: chosen parameter-parameter_set combination that is collected
Returns:
pandas.DataFrame for given parameter of station
"""
parameter, parameter_set = parameter
periods_and_date_ranges = []
for period in self.periods:
if (self.resolution in HIGH_RESOLUTIONS
and period == DWDObservationPeriod.HISTORICAL):
date_ranges = self._get_historical_date_ranges(
station_id, parameter_set)
for date_range in date_ranges:
periods_and_date_ranges.append((period, date_range))
else:
periods_and_date_ranges.append((period, None))
parameter_df = pd.DataFrame()
for period, date_range in periods_and_date_ranges:
parameter_identifier = build_parameter_set_identifier(
parameter_set, self.resolution, period, station_id, date_range)
log.info(
f"Acquiring observations data for {parameter_identifier}.")
# TODO: integrate collect_climate_observations_data in class
try:
period_df = collect_climate_observations_data(
station_id, parameter_set, self.resolution, period,
date_range)
except InvalidParameterCombination:
log.info(f"Invalid combination {parameter_set.value}/"
f"{self.resolution.value}/{period} is skipped.")
period_df = pd.DataFrame()
# Filter out values which already are in the DataFrame
try:
period_df = period_df[~period_df[DWDMetaColumns.DATE.value].
isin(parameter_df[DWDMetaColumns.DATE.
value])]
except KeyError:
pass
parameter_df = parameter_df.append(period_df)
if self.tidy_data:
parameter_df = parameter_df.dwd.tidy_up_data()
# TODO: remove this column and rather move it into metadata of resulting
# data model
parameter_df.insert(2, DWDMetaColumns.PARAMETER_SET.value,
parameter_set.name)
parameter_df[DWDMetaColumns.PARAMETER_SET.value] = parameter_df[
DWDMetaColumns.PARAMETER_SET.value].astype("category")
if parameter not in DWDObservationParameterSet:
parameter_df = parameter_df[parameter_df[
DWDMetaColumns.PARAMETER.value] == parameter.value]
return parameter_df