def imputation(data, threshold="3600S"):
"""
Goes through the dataset and saves a data point and his predecessor in a dictionary if
the time difference between them is bigger than a given threshold
Name in documentation: imputation
:param data: The data that should be checked as Pandas.Series
:param threshold: The threshold given in second
:return: The data points as a dictionary
:exception NoDataException: Exception is thrown when data is empty
:exception InvalidConfigValueException: Exception which is thrown if the threshold is to low.
"""
if data.empty:
raise exc.NoDataException("Data mustn't be empty")
if int(threshold[:-1]) < 1:
raise exc.InvalidConfigValueException(
'Threshold must be greater than 1.')
imp_frame = pd.DataFrame(columns=["from", "to"])
time_index = pd.Series(data.index)
time_delta = time_index.diff()
threshold = pd.to_timedelta(threshold)
froms = []
tos = []
for i in range(1, len(time_delta)):
if pd.to_timedelta(time_delta.values[i]) > threshold:
froms.append(time_index.get(i - 1))
tos.append(time_index.get(i))
imp_frame["from"] = froms
imp_frame["to"] = tos
return imp_frame
def get_data(from_db, from_measurement, value_name, register, time):
"""
Retrieve data from database
Name in documentation: get_data
:param from_db: database from which the data should be extracted
:param from_measurement: the measurement from which the data should be extracted
:param value_name: name of the field in influxdb
:param register: register of the desired curve which should be cleaned
:param time: tart and end time of the desired period where the curve should be cleaned
:return: returns temperature data as a Pandas.Series
"""
query = ""
if isInt(register):
query = "SELECT {0} FROM {1} WHERE register='{2}' and time >= {3}ms and time <= {4}ms".format(
value_name, from_measurement, register, time["from"], time["to"])
else:
query = "SELECT {0} FROM {1} WHERE time >={2}ms AND time <={3}ms".format(
value_name, from_measurement, time["from"], time["to"])
try:
res = rm.read_query(from_db, query)
except KeyError as ke:
raise exc.NoDataException(
"Result from Database was probably empty. Machste erstmal die basics"
)
return res
def check_if_request_empty():
""" Checks if the incoming http-post-request comes with an empty body.
:raise InvalidConfigException if the body is empty.
:return request-object.
"""
if int(request.headers.get('Content-Length')) == 0:
raise exceptions.NoDataException('Message must not be empty', 900)
return request.get_json(force=True)
def interpolation(data):
"""
Method to interpolate the given data (resampled data) with an cubic interpolation, which is used to fill the gaps
made in the resample() method
Name in documentation: interpolation
:param data: given data as a pandas Series which should be interpolated in this method, should be resampled before
:return data.interpolate(): interpolated data without gaps.
:exception NoDataException: Exception is thrown when data is empty
"""
if data.empty:
raise exc.NoDataException("Data mustn't be empty")
return data.interpolate(method='cubic')
def resample(data, freq='60S'):
"""
Method to resample the given data with the given frequency. This is necessary for an union time intervall.
Name in documentation: resample
:param data: given data as pandas Series which should be resampled in this method.
:param freq: A frequency, needed for the resampling, it is used to generate the union time intervall.
:return dsata.resample(freq).asfreq(): resampled data with an union time intervall as a pandas Series.
:exception NoDataException: Exception is thrown when data is empty
:exception InvalidConfigValueException: Exception which is thrown if the frequency is to low.
"""
if data.empty:
raise exc.NoDataException("Data mustn't be empty")
if int(freq[:-1]) < 1:
raise exc.InvalidConfigValueException(
'Resample frequency cannot be less than 1S')
data = data[~data.index.duplicated()]
return data.resample(freq).asfreq()
def rolling_mean(data, frame_width=100):
"""
Name in documentation: rolling_mean()
Applies the rolling mean on the data based on a given frame width
Name in documentation: rolling_mean
:param data: The data on which the rolling mean should be applied on as a pandas Series
:param frame_width: The window width that should be used for the rolling mean
:return: The data on which the rolling mean was applied on
:exception NoDataException: Exception is thrown when data is empty
:exception InvalidConfigValueException: Exception which is thrown if the framewidth is to low.
"""
if data.empty:
raise exc.NoDataException("Data mustn't be empty")
if frame_width < 0:
raise exc.InvalidConfigValueException(
'Framewidth cannot be less than 0 ')
return data.rolling(frame_width, center=True).mean()
def remove_gaps(data, imp_frame):
"""
Method to remove the gaps of the given data which are found out in the imputation() method
:param data: whole interpolated data as pandas Series in which the gaps should be cutted
:param imputation_dict: dictionary of gaps which contains the timestamps of the found gaps. If no gaps were found,
remove_gaps has no effect.
:return data: cut data with NaN timestamps at the given time intervalls from the imputation_dict
:exception NoDataException: Exception is thrown when data is empty
:exception ImputationDictionaryException: Exception is thrown when no Imputation Dictionary is given
"""
# Using the implicit booleanness of the empty list is quite pythonic
if data.empty:
raise exc.NoDataException("Data mustn't be empty")
if imp_frame is None:
raise exc.ImputationDictionaryException(
"Imputation Dictionary mustn't be None")
# set values between gaps to NaN with cut function
for i in range(len(imp_frame.index)):
# cut(data, imputation_dict["gap" + str(i)]["from"], imputation_dict["gap" + str(i)]["to"])
data = cut(data, imp_frame["from"].get(i), imp_frame["to"].get(i))
return data.astype('float64')