def __init__(self, columns_to_interpolate: List[str], timestamp_round_algorithm: AbstractTimestampRoundAlgorithm, timestamp_interpolation_algorithm: AbstractTimestampInterpolationAlgorithm, border_values_interpolation_algorithm: AbstractValueInterpolationAlgorithm, internal_values_interpolation_algorithm: AbstractValueInterpolationAlgorithm, timestamp_filter_algorithm: AbstractTimestampFilterAlgorithm = LeftClosedTimestampFilterAlgorithm() ) -> None: self._columns_to_process = columns_to_interpolate self._timestamp_round_algorithm = timestamp_round_algorithm self._timestamp_interpolation_algorithm = timestamp_interpolation_algorithm self._border_values_interpolation_algorithm = border_values_interpolation_algorithm self._internal_values_interpolation_algorithm = internal_values_interpolation_algorithm self._timestamp_filter_algorithm = timestamp_filter_algorithm logger.debug( f"Creating instance:" f"columns_to_interpolate: {self._columns_to_process}" f"timestamp_round_algorithm: {self._timestamp_round_algorithm}" f"timestamp_interpolation_algorithm: {self._timestamp_interpolation_algorithm}" f"border_values_interpolation_algorithm: {self._border_values_interpolation_algorithm}" f"internal_values_interpolation_algorithm: {self._internal_values_interpolation_algorithm}" f"timestamp_filter_algorithm: {self._timestamp_filter_algorithm}" )
def _parse_timestamp(self, df: pd.DataFrame) -> pd.DataFrame: logger.debug("Parsing datetime") df = df.copy() df[column_names.TIMESTAMP] = df[column_names.TIMESTAMP].apply( parse_datetime, args=(self._timestamp_parse_patterns, self._timestamp_timezone) ) return df
def _convert_values_to_float(self, df: pd.DataFrame) -> pd.DataFrame: logger.debug("Converting values to float") df = df.copy() for column_name in self._float_columns: df[column_name] = df[column_name].str.replace(",", ".", regex=False) df[column_name] = df[column_name].apply(float) return df
def __init__(self, timestamp_parse_patterns: List[str], timestamp_timezone, need_columns: List[str], float_columns: List[str], water_temp_columns: List[str], need_circuit: str, encoding: str = "utf-8" ) -> None: self._encoding = encoding self._timestamp_timezone = timestamp_timezone self._timestamp_parse_patterns = timestamp_parse_patterns self._need_circuit = need_circuit self._need_columns = need_columns self._float_columns = float_columns self._water_temp_columns = water_temp_columns self._circuit_id_equals = soft_m_circuit_ids_equal.CIRCUIT_ID_EQUALS self._column_names_equals = processing_parameters.HEATING_OBJ_COLUMN_NAMES_EQUALS logger.debug( f"Creating instance:" f"encoding: {self._encoding}" f"timestamp_timezone: {self._timestamp_timezone}" f"timestamp_parse_patterns: {self._timestamp_parse_patterns}" f"need_circuit: {self._need_circuit}" f"need_columns: {self._need_columns}" f"float_columns: {self._float_columns}" f"water_temp_columns: {self._water_temp_columns}" )
def _divide_incorrect_hot_water_temp(self, df: pd.DataFrame) -> pd.DataFrame: logger.debug("Dividing incorrect water temp") df = df.copy() for column_name in self._water_temp_columns: df[column_name] = df[column_name].apply( lambda water_temp: water_temp > 100 and water_temp / 100 or water_temp ) return df
def read_temp_graph_from_binary_stream(self, binary_stream: BinaryIO ) -> pd.DataFrame: logger.debug("Reading temp graph") df = pd.read_json(binary_stream, encoding=self._encoding) df = self._rename_columns(df) logger.debug("Temp graph is read") return df
def _rename_equal_columns(self, df: pd.DataFrame) -> pd.DataFrame: logger.debug("Renaming equal columns") df = df.copy() column_names_equals = {} for soft_m_column_name, target_column_name in self._column_names_equals.items(): if target_column_name in self._need_columns: column_names_equals[soft_m_column_name] = target_column_name df = df.rename(columns=column_names_equals) return df
def read_weather_from_binary_stream( self, binary_stream: BinaryIO) -> pd.DataFrame: logger.debug("Parsing weather") with io.TextIOWrapper(binary_stream, encoding=self._encoding) as text_stream: df = pd.read_json(text_stream, convert_dates=False) self._rename_columns(df) self._convert_date_and_time_to_timestamp(df) logger.debug("Weather is parsed") return df
def __init__(self, encoding: str = "utf-8" ) -> None: self._encoding = encoding self._column_names_equal = processing_parameters.TEMP_GRAPH_COLUMN_NAMES_EQUALS logger.debug( f"Creating instance:" f"encoding: {encoding}" )
async def _get_temp_graph_from_server(self) -> bytes: url = f"{self._temp_graph_server_address}/JSON" params = { "method": "getTempGraphic" } async with aiohttp.request("GET", url=url, params=params, proxy=self._http_proxy) as response: raw_response = await response.read() logger.debug(f"Temp graph is loaded from server. " f"Response status code is {response.status}") return raw_response
def __init__(self, encoding: str = "utf-8", weather_data_timezone=None) -> None: self._weather_data_timezone = weather_data_timezone self._encoding = encoding self._column_names_equals = boiler_softm.constants.processing_parameters.WEATHER_INFO_COLUMN_EQUALS logger.debug(f"Creating instance:" f"weather_data_timezone: {self._weather_data_timezone}" f"encoding: {self._encoding}")
def _convert_date_and_time_to_timestamp(self, df: pd.DataFrame) -> None: logger.debug("Converting dates and time to timestamp") dates_as_str = df[soft_m_column_names.WEATHER_DATE] time_as_str = df[soft_m_column_names.WEATHER_TIME] datetime_as_str = dates_as_str.str.cat(time_as_str, sep=" ") timestamp = pd.to_datetime(datetime_as_str) timestamp = timestamp.dt.tz_localize(self._weather_data_timezone) df[column_names.TIMESTAMP] = timestamp del df[soft_m_column_names.WEATHER_TIME] del df[soft_m_column_names.WEATHER_DATE]
async def load_weather( self, start_datetime: Optional[pd.Timestamp] = None, end_datetime: Optional[pd.Timestamp] = None) -> pd.DataFrame: logger.debug( f"Requested weather forecast from {start_datetime} to {end_datetime}" ) raw_weather_forecast = await self._get_forecast_from_server() weather_df = await self._read_weather_forecast(raw_weather_forecast) weather_df = self._filter_by_timestamp(end_datetime, start_datetime, weather_df) logger.debug(f"Gathered {len(weather_df)} weather forecast items") return weather_df
async def _get_forecast_from_server(self) -> bytes: url = f"{self._weather_data_server_address}/JSON" # noinspection SpellCheckingInspection params = {"method": "getPrognozT"} async with aiohttp.request("GET", url=url, params=params, proxy=self._http_proxy) as response: raw_response = await response.read() logger.debug(f"Weather forecast is loaded. " f"Response status code is {response.status}") return raw_response
def process_heating_obj(self, heating_obj_df: pd.DataFrame, min_required_timestamp: Union[pd.Timestamp, None], max_required_timestamp: Union[pd.Timestamp, None] ) -> pd.DataFrame: logger.debug(f"Processing heating obj {min_required_timestamp}, {max_required_timestamp}") heating_obj_df = heating_obj_df.copy() heating_obj_df = self._round_timestamp(heating_obj_df) heating_obj_df = self._drop_duplicates_by_timestamp(heating_obj_df) heating_obj_df = self._interpolate_timestamp(heating_obj_df, max_required_timestamp, min_required_timestamp) heating_obj_df = self._interpolate_values(heating_obj_df) heating_obj_df = self._filter_by_timestamp(heating_obj_df, max_required_timestamp, min_required_timestamp) return heating_obj_df
def __init__(self, reader: AbstractSyncTempGraphReader, server_address: str = "https://lysva.agt.town", http_proxy: Optional[str] = None, sync_executor: Optional[ThreadPoolExecutor] = None ) -> None: self._temp_graph_reader = reader self._temp_graph_server_address = server_address self._http_proxy = http_proxy self._sync_executor = sync_executor logger.debug( f"Creating instance: " f"reader: {self._temp_graph_reader} " f"server_address: {self._temp_graph_server_address} " f"http_proxy: {self._http_proxy} " f"sync_executor: {self._sync_executor} " )
def __init__( self, reader: AbstractSyncWeatherReader, timestamp_filter_algorithm: AbstractTimestampFilterAlgorithm = LeftClosedTimestampFilterAlgorithm( ), server_address: str = "https://lysva.agt.town", http_proxy: Optional[str] = None, sync_executor: ThreadPoolExecutor = None) -> None: self._weather_reader = reader self._weather_data_server_address = server_address self._timestamp_filter_algorithm = timestamp_filter_algorithm self._http_proxy = http_proxy self._sync_executor = sync_executor logger.debug( f"Creating instance: " f"reader: {self._weather_reader} " f"server_address: {self._weather_data_server_address} " f"timestamp_filter_algorithm: {self._timestamp_filter_algorithm} " f"http_proxy: {self._http_proxy} " f"sync_executor: {self._sync_executor} ")
def read_heating_obj_from_binary_stream(self, binary_stream: BinaryIO ) -> pd.DataFrame: logger.debug("Loading heating obj data") df = pd.read_csv( binary_stream, sep=";", low_memory=False, parse_dates=False, encoding=self._encoding ) logger.debug("Parsing heating obj data") df = self._exclude_unused_circuits(df) df = self._rename_equal_columns(df) df = self._parse_timestamp(df) df = self._convert_values_to_float(df) df = self._divide_incorrect_hot_water_temp(df) df = self._exclude_unused_columns(df) return df
def _rename_columns(self, df: pd.DataFrame ) -> pd.DataFrame: logger.debug("Renaming columns") df = df.rename(columns=self._column_names_equal).copy() return df
def _rename_columns(self, df: pd.DataFrame) -> None: logger.debug("Renaming columns") df.rename(columns=self._column_names_equals, inplace=True)
async def load_temp_graph(self) -> pd.DataFrame: logger.debug("Loading temp graph") temp_graph_as_bytes = await self._get_temp_graph_from_server() temp_graph_df = await self._read_temp_graph(temp_graph_as_bytes) return temp_graph_df
def _exclude_unused_circuits(self, df: pd.DataFrame) -> pd.DataFrame: logger.debug("Excluding unused circuits") renamed_circuits = \ df[soft_m_column_names.HEATING_SYSTEM_CIRCUIT_ID].apply(str).replace(self._circuit_id_equals) df = df[renamed_circuits == self._need_circuit].copy() return df
def _exclude_unused_columns(self, df: pd.DataFrame) -> pd.DataFrame: logger.debug("Excluding unused columns") df = df[self._need_columns].copy() return df