def filter(config, timeframe):
'''
Name in documentation: 'filtern'
Takes the config and load the klassified data. After that it delete and interpolate the marked intervall. Last it persist the filtered data.
:raises ConfigExeption: For incorrect Config.
'''
filtern_data = get_data(timeframe)
logger.info("Get_data was successful")
try:
for curve in config:
for cycle in config[curve]:
if config[curve][cycle]["delete"] == "True":
filtern_data = tag_drop(curve, cycle, filtern_data)
method = config[curve][cycle]["Interpolation"]
filtern_data = interpolation(method, curve, filtern_data)
#print("Gefilterte Daten:")
#print(filtern_data.loc[filtern_data.abtauzyklus == True])
logger.info("tag_drop and interpolation was successful")
except exe.ConfigException:
logger.warning()
raise exe.ConfigException("Filtern Config format is not correct.", 900)
persist_data(filtern_data)
logger.info("Persisted filterd data successfully")
def get_config_parameter(config):
"""
Extract relevant parameters from the config dictionary
:param config: Dictionary from which the parameters will be extracted
:raises ConfigException: Raised if parameters from the config are wrong or missing
:return datasource_enriched_data: Database from which valuescalculated from the data are loaded
:return datasource_classified_data: Database for the classified data
:return datasource_raw_data Database: from which data are loaded to be classified
:return timeframe: Period of time
:return selected_event: Holds the value for the type of event ("Standard" or "Prediction")
:return measurement_classified: Measurement from Database of the selected event
:return datasource_raw_data: Database that contains raw data
:return measurement_raw: Measurement from Database that contains raw data
:return events: Possible events which are then classified
:return measurement_enriched: Measurement from Database that contains enriched data
:return datasource_predicted_data: Database that contains predicted data
:return measurement_predicted: Measurement from Database that contains the predicted data
"""
try:
datasource_enriched_data = config['datasource_enriched_data'][
'database']
datasource_classified_data = config['datasource_classified_data'][
'database']
timeframe = config['timeframe']
selected_event = config['selected_event']
measurement_classified = config['datasource_classified_data'][
'measurement']
datasource_raw_data = config['datasource_raw_data']['database']
measurement_raw = config['datasource_raw_data']['measurement']
events = config[selected_event]
measurement_enriched = config['datasource_enriched_data'][
'measurement']
datasource_predicted_data = config['datasource_predicted_data'][
'database']
measurement_predicted = config['datasource_predicted_data'][
'measurement']
except Exception:
raise ex.ConfigException("Missing or wrong parameters in config")
return datasource_enriched_data, \
datasource_classified_data, \
timeframe, \
selected_event, \
measurement_classified, \
datasource_raw_data, \
measurement_raw, \
events, \
measurement_enriched,\
datasource_predicted_data, \
measurement_predicted
def interpolation(methode, curve, zyklenfreie_daten):
'''
Name in documentation: 'interpolation'
Interpolate one curve, after a cycle was deletet. The method has already been read from config file.
:param methode: The interpolation method as string.
:param kurve: the name of the curve as string.
:param zyklenfreie_daten: the klassified data with deleted fields.
:raises ConfigExeption: If the config is wrong.
:return: The klassified data after filtering a cycle.
'''
try:
zyklenfreie_daten[curve] = zyklenfreie_daten[curve].interpolate(method= methode, order = 3)
print("Interpoliert:")
#print(zyklenfreie_daten.loc[zyklenfreie_daten.abtauzyklus == True][curve])
#print("________________________________")
except exe.DataPipelineException as e:
logger.error(e.message)
raise exe.ConfigException("Filtern Config format is not correct. Interpolation() failed.", 900)
return zyklenfreie_daten
def tag_drop(curve, cycle, filtern_data):
'''
Name in documentation: 'tag_drop'
Delete one cycle in one curve.
:param curve: the name of the curve as string
:param cycle: the name of the cycle as string
:param filtern_data: the klassified data
:raises ConfigExeption: If the config is wrong.
:return: the klassified data with one cycle of one curve deleted
'''
try:
filtern_data.loc[filtern_data[cycle] == 1, curve] = np.NaN
#print("Tag_Drop:")
#print(curve)
#print(filtern_data.loc[filtern_data.abtauzyklus == True][curve])
#print("________________________________")
except:
raise exe.ConfigException("Filtern Config format is not correct. Tag_drop() failed", 900)
return filtern_data
def train_classifier(config):
"""
Name in documentation: klassifizierer_trainieren()
Train a classifier to identify a specific event.
:param classification_config: Contains parameters for training the classifier
:param classifier:(sklearn-object) a classification algorithm form the sklearn package
:raises InvalidConfigValueException:Raised if a value inside of the config is wrong
:raises PersistorException: Raised if classifier is not an instance of sklearn
:return int: Status code that indicates whether the training was successful(0 Success, 1 Failure)"""
try:
selected_event, required_score, test_size, datasource_marked_data, start_time, end_time, events = get_config_parameter(
config)
except Exception:
raise exce.InvalidConfigValueException
logger.info("config parameter loaded")
try:
start = convert_time(start_time)
end = convert_time(end_time)
except Exception as e:
raise exce.InvalidConfigValueException(str(e))
df = read_manager.read_query(
datasource_marked_data,
f"SELECT * FROM {selected_event} WHERE time >= {start}ms AND time <= {end}ms"
)
for event in events:
end_start = markers[event]
start_event = list(end_start.keys())[0]
end_event = list(end_start.values())[len(end_start) - 1]
if (str(df.index[0]) > start_event) or (str(df.index[-1]) < end_event):
raise exce.ConfigException(
'time frame of trainingsdata not in selected data frame included'
)
df_copy = df.copy()[start_event:end_event]
try:
classifier = model_persistor.load_classifier(config, event, True)
except Exception as e:
raise exce.PersistorException(str(e))
logger.info("model loaded")
df_copy.dropna(inplace=True)
y = np.array(df_copy[event])
for drop_event in events:
df_copy = df_copy.drop(labels=[drop_event, f"{drop_event}_marker"],
axis=1)
X = df_copy.to_numpy()
try:
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=test_size)
except Exception as e:
raise exce.SklearnException(str(e))
try:
classifier = classifier.fit(X_train, y_train)
except Exception as e:
raise exce.SklearnException(str(e))
logger.info("model trained")
if evaluate_classifier(classifier, required_score[event], X_test,
y_test):
model_persistor.persist_classifier(classifier, config, event)
logger.info('model persisted')
else:
logger.info('score too low, model not persisted')
return 0
def config_validation(filtern_config):
"""
Name in documentation: 'config_validation'
Validate the config. It is checked whether the values for "delete" are only "True" and "False"
and whether the values for "Interpolation" are only "linear", "cubic", "spline" and "akima".
Also checks if every curve and cycle is in the config.
If this is not the case, an ConfigException is thrown.
:raises: ConfigExeption: For incorrect Config.
:raises: InvalidConfigKeyException: For wrong keys in Config.
:raises: InvalidConfigValueException: For wrong Values in Config.
:raises: IncompleteConfigException: For missing anything in Config.
:param filtern_config: The filtern_config:
"""
try:
config = filtern_config["filter_options"][
filtern_config["selected_value"]]
timeframe = filtern_config['timeframe']
except:
raise exe.InvalidConfigKeyException("Can not read Filtern Config.",
900)
expected_curve = [
'room', 'condenser', 'evaporator', 'inlet', 'outlet', 'freshAirIntake'
]
for curve in config:
if curve in expected_curve:
expected_curve.remove(curve)
expected_cycle = [
'warmwasseraufbereitung', 'ofennutzung', 'abtauzyklus',
'luefterstufen'
]
for cycle in config[curve]:
expected_delete_interpolation = ['delete', 'Interpolation']
for delete_interpolation in config[curve][cycle]:
if delete_interpolation in expected_delete_interpolation:
expected_delete_interpolation.remove(delete_interpolation)
if expected_delete_interpolation != []:
raise exe.InvalidConfigKeyException(
"Filtern Config got no Interpolation or Delete.", 900)
if cycle in expected_cycle:
expected_cycle.remove(cycle)
if config[curve][cycle]["delete"] != 'True' and config[curve][
cycle]["delete"] != 'False':
raise exe.InvalidConfigValueException(
"Filtern Config Delete is not True or False.", 900)
if config[curve][cycle]["Interpolation"] != 'linear' and config[
curve][cycle]["Interpolation"] != 'cubic' and config[
curve][cycle]["Interpolation"] != 'spline' and config[
curve][cycle]["Interpolation"] != 'akima':
raise exe.InvalidConfigValueException(
"Filtern Config Interpolation is not linear, cubic, spline or akima.",
900)
if expected_cycle != []:
raise exe.IncompleteConfigException(
"Filtern Config missing a cycle.", 900)
if expected_curve != []:
raise exe.IncompleteConfigException("Filtern Config missing a curve.",
900)
for time in timeframe:
if not re.search(
r"[0-9][0-9][0-9][0-9][-][0-1][0-9][-][0-3][0-9][ ][0-2][0-9][:][0-6][0-9][:][0-6][0-9][.][0-9][0-9][0-9][ ][U][T][C]",
time):
raise exe.ConfigException(
"Filtern Config Timeframe Format is not correct.", 900)