def __init__(self,
columns_to_interpolate: List[str],
timestamp_round_algorithm: AbstractTimestampRoundAlgorithm,
timestamp_interpolation_algorithm: AbstractTimestampInterpolationAlgorithm,
border_values_interpolation_algorithm: AbstractValueInterpolationAlgorithm,
internal_values_interpolation_algorithm: AbstractValueInterpolationAlgorithm,
timestamp_filter_algorithm: AbstractTimestampFilterAlgorithm =
LeftClosedTimestampFilterAlgorithm()
) -> None:
self._columns_to_process = columns_to_interpolate
self._timestamp_round_algorithm = timestamp_round_algorithm
self._timestamp_interpolation_algorithm = timestamp_interpolation_algorithm
self._border_values_interpolation_algorithm = border_values_interpolation_algorithm
self._internal_values_interpolation_algorithm = internal_values_interpolation_algorithm
self._timestamp_filter_algorithm = timestamp_filter_algorithm
logger.debug(
f"Creating instance:"
f"columns_to_interpolate: {self._columns_to_process}"
f"timestamp_round_algorithm: {self._timestamp_round_algorithm}"
f"timestamp_interpolation_algorithm: {self._timestamp_interpolation_algorithm}"
f"border_values_interpolation_algorithm: {self._border_values_interpolation_algorithm}"
f"internal_values_interpolation_algorithm: {self._internal_values_interpolation_algorithm}"
f"timestamp_filter_algorithm: {self._timestamp_filter_algorithm}"
)
def _parse_timestamp(self, df: pd.DataFrame) -> pd.DataFrame:
logger.debug("Parsing datetime")
df = df.copy()
df[column_names.TIMESTAMP] = df[column_names.TIMESTAMP].apply(
parse_datetime, args=(self._timestamp_parse_patterns, self._timestamp_timezone)
)
return df
def _convert_values_to_float(self, df: pd.DataFrame) -> pd.DataFrame:
logger.debug("Converting values to float")
df = df.copy()
for column_name in self._float_columns:
df[column_name] = df[column_name].str.replace(",", ".", regex=False)
df[column_name] = df[column_name].apply(float)
return df
def __init__(self,
timestamp_parse_patterns: List[str],
timestamp_timezone,
need_columns: List[str],
float_columns: List[str],
water_temp_columns: List[str],
need_circuit: str,
encoding: str = "utf-8"
) -> None:
self._encoding = encoding
self._timestamp_timezone = timestamp_timezone
self._timestamp_parse_patterns = timestamp_parse_patterns
self._need_circuit = need_circuit
self._need_columns = need_columns
self._float_columns = float_columns
self._water_temp_columns = water_temp_columns
self._circuit_id_equals = soft_m_circuit_ids_equal.CIRCUIT_ID_EQUALS
self._column_names_equals = processing_parameters.HEATING_OBJ_COLUMN_NAMES_EQUALS
logger.debug(
f"Creating instance:"
f"encoding: {self._encoding}"
f"timestamp_timezone: {self._timestamp_timezone}"
f"timestamp_parse_patterns: {self._timestamp_parse_patterns}"
f"need_circuit: {self._need_circuit}"
f"need_columns: {self._need_columns}"
f"float_columns: {self._float_columns}"
f"water_temp_columns: {self._water_temp_columns}"
)
def _divide_incorrect_hot_water_temp(self, df: pd.DataFrame) -> pd.DataFrame:
logger.debug("Dividing incorrect water temp")
df = df.copy()
for column_name in self._water_temp_columns:
df[column_name] = df[column_name].apply(
lambda water_temp: water_temp > 100 and water_temp / 100 or water_temp
)
return df
def read_temp_graph_from_binary_stream(self,
binary_stream: BinaryIO
) -> pd.DataFrame:
logger.debug("Reading temp graph")
df = pd.read_json(binary_stream, encoding=self._encoding)
df = self._rename_columns(df)
logger.debug("Temp graph is read")
return df
def _rename_equal_columns(self, df: pd.DataFrame) -> pd.DataFrame:
logger.debug("Renaming equal columns")
df = df.copy()
column_names_equals = {}
for soft_m_column_name, target_column_name in self._column_names_equals.items():
if target_column_name in self._need_columns:
column_names_equals[soft_m_column_name] = target_column_name
df = df.rename(columns=column_names_equals)
return df
def read_weather_from_binary_stream(
self, binary_stream: BinaryIO) -> pd.DataFrame:
logger.debug("Parsing weather")
with io.TextIOWrapper(binary_stream,
encoding=self._encoding) as text_stream:
df = pd.read_json(text_stream, convert_dates=False)
self._rename_columns(df)
self._convert_date_and_time_to_timestamp(df)
logger.debug("Weather is parsed")
return df
def __init__(self,
encoding: str = "utf-8"
) -> None:
self._encoding = encoding
self._column_names_equal = processing_parameters.TEMP_GRAPH_COLUMN_NAMES_EQUALS
logger.debug(
f"Creating instance:"
f"encoding: {encoding}"
)
async def _get_temp_graph_from_server(self) -> bytes:
url = f"{self._temp_graph_server_address}/JSON"
params = {
"method": "getTempGraphic"
}
async with aiohttp.request("GET", url=url, params=params, proxy=self._http_proxy) as response:
raw_response = await response.read()
logger.debug(f"Temp graph is loaded from server. "
f"Response status code is {response.status}")
return raw_response
def __init__(self,
encoding: str = "utf-8",
weather_data_timezone=None) -> None:
self._weather_data_timezone = weather_data_timezone
self._encoding = encoding
self._column_names_equals = boiler_softm.constants.processing_parameters.WEATHER_INFO_COLUMN_EQUALS
logger.debug(f"Creating instance:"
f"weather_data_timezone: {self._weather_data_timezone}"
f"encoding: {self._encoding}")
def _convert_date_and_time_to_timestamp(self, df: pd.DataFrame) -> None:
logger.debug("Converting dates and time to timestamp")
dates_as_str = df[soft_m_column_names.WEATHER_DATE]
time_as_str = df[soft_m_column_names.WEATHER_TIME]
datetime_as_str = dates_as_str.str.cat(time_as_str, sep=" ")
timestamp = pd.to_datetime(datetime_as_str)
timestamp = timestamp.dt.tz_localize(self._weather_data_timezone)
df[column_names.TIMESTAMP] = timestamp
del df[soft_m_column_names.WEATHER_TIME]
del df[soft_m_column_names.WEATHER_DATE]
async def load_weather(
self,
start_datetime: Optional[pd.Timestamp] = None,
end_datetime: Optional[pd.Timestamp] = None) -> pd.DataFrame:
logger.debug(
f"Requested weather forecast from {start_datetime} to {end_datetime}"
)
raw_weather_forecast = await self._get_forecast_from_server()
weather_df = await self._read_weather_forecast(raw_weather_forecast)
weather_df = self._filter_by_timestamp(end_datetime, start_datetime,
weather_df)
logger.debug(f"Gathered {len(weather_df)} weather forecast items")
return weather_df
async def _get_forecast_from_server(self) -> bytes:
url = f"{self._weather_data_server_address}/JSON"
# noinspection SpellCheckingInspection
params = {"method": "getPrognozT"}
async with aiohttp.request("GET",
url=url,
params=params,
proxy=self._http_proxy) as response:
raw_response = await response.read()
logger.debug(f"Weather forecast is loaded. "
f"Response status code is {response.status}")
return raw_response
def process_heating_obj(self,
heating_obj_df: pd.DataFrame,
min_required_timestamp: Union[pd.Timestamp, None],
max_required_timestamp: Union[pd.Timestamp, None]
) -> pd.DataFrame:
logger.debug(f"Processing heating obj {min_required_timestamp}, {max_required_timestamp}")
heating_obj_df = heating_obj_df.copy()
heating_obj_df = self._round_timestamp(heating_obj_df)
heating_obj_df = self._drop_duplicates_by_timestamp(heating_obj_df)
heating_obj_df = self._interpolate_timestamp(heating_obj_df, max_required_timestamp, min_required_timestamp)
heating_obj_df = self._interpolate_values(heating_obj_df)
heating_obj_df = self._filter_by_timestamp(heating_obj_df, max_required_timestamp, min_required_timestamp)
return heating_obj_df
def __init__(self,
reader: AbstractSyncTempGraphReader,
server_address: str = "https://lysva.agt.town",
http_proxy: Optional[str] = None,
sync_executor: Optional[ThreadPoolExecutor] = None
) -> None:
self._temp_graph_reader = reader
self._temp_graph_server_address = server_address
self._http_proxy = http_proxy
self._sync_executor = sync_executor
logger.debug(
f"Creating instance: "
f"reader: {self._temp_graph_reader} "
f"server_address: {self._temp_graph_server_address} "
f"http_proxy: {self._http_proxy} "
f"sync_executor: {self._sync_executor} "
)
def __init__(
self,
reader: AbstractSyncWeatherReader,
timestamp_filter_algorithm:
AbstractTimestampFilterAlgorithm = LeftClosedTimestampFilterAlgorithm(
),
server_address: str = "https://lysva.agt.town",
http_proxy: Optional[str] = None,
sync_executor: ThreadPoolExecutor = None) -> None:
self._weather_reader = reader
self._weather_data_server_address = server_address
self._timestamp_filter_algorithm = timestamp_filter_algorithm
self._http_proxy = http_proxy
self._sync_executor = sync_executor
logger.debug(
f"Creating instance: "
f"reader: {self._weather_reader} "
f"server_address: {self._weather_data_server_address} "
f"timestamp_filter_algorithm: {self._timestamp_filter_algorithm} "
f"http_proxy: {self._http_proxy} "
f"sync_executor: {self._sync_executor} ")
def read_heating_obj_from_binary_stream(self,
binary_stream: BinaryIO
) -> pd.DataFrame:
logger.debug("Loading heating obj data")
df = pd.read_csv(
binary_stream,
sep=";",
low_memory=False,
parse_dates=False,
encoding=self._encoding
)
logger.debug("Parsing heating obj data")
df = self._exclude_unused_circuits(df)
df = self._rename_equal_columns(df)
df = self._parse_timestamp(df)
df = self._convert_values_to_float(df)
df = self._divide_incorrect_hot_water_temp(df)
df = self._exclude_unused_columns(df)
return df
def _rename_columns(self,
df: pd.DataFrame
) -> pd.DataFrame:
logger.debug("Renaming columns")
df = df.rename(columns=self._column_names_equal).copy()
return df
def _rename_columns(self, df: pd.DataFrame) -> None:
logger.debug("Renaming columns")
df.rename(columns=self._column_names_equals, inplace=True)
async def load_temp_graph(self) -> pd.DataFrame:
logger.debug("Loading temp graph")
temp_graph_as_bytes = await self._get_temp_graph_from_server()
temp_graph_df = await self._read_temp_graph(temp_graph_as_bytes)
return temp_graph_df
def _exclude_unused_circuits(self, df: pd.DataFrame) -> pd.DataFrame:
logger.debug("Excluding unused circuits")
renamed_circuits = \
df[soft_m_column_names.HEATING_SYSTEM_CIRCUIT_ID].apply(str).replace(self._circuit_id_equals)
df = df[renamed_circuits == self._need_circuit].copy()
return df
def _exclude_unused_columns(self, df: pd.DataFrame) -> pd.DataFrame:
logger.debug("Excluding unused columns")
df = df[self._need_columns].copy()
return df