def subtract_min_without_nan(segment: pd.Series) -> pd.Series:
if len(segment) == 0:
return []
nan_list = utils.find_nan_indexes(segment)
if len(nan_list) > 0:
return segment
else:
segment = segment - min(segment)
return segment
def __filter_detection(self, segments_indexes: List[int], data: list):
delete_list = []
variance_error = self.state.window_size
close_segments = utils.close_filtering(segments_indexes,
variance_error)
segments_indexes = utils.best_pattern(close_segments, data,
self.get_extremum_type().value)
if len(segments_indexes) == 0 or len(self.state.pattern_center) == 0:
return []
pattern_data = self.state.pattern_model
for segment_index in segments_indexes:
if segment_index <= self.state.window_size or segment_index >= (
len(data) - self.state.window_size):
delete_list.append(segment_index)
continue
convol_data = utils.get_interval(data, segment_index,
self.state.window_size)
percent_of_nans = convol_data.isnull().sum() / len(convol_data)
if len(convol_data) == 0 or percent_of_nans > 0.5:
delete_list.append(segment_index)
continue
elif 0 < percent_of_nans <= 0.5:
nan_list = utils.find_nan_indexes(convol_data)
convol_data = utils.nan_to_zero(convol_data, nan_list)
pattern_data = utils.nan_to_zero(pattern_data, nan_list)
conv = scipy.signal.fftconvolve(convol_data, pattern_data)
if len(conv) == 0:
delete_list.append(segment_index)
continue
upper_bound = self.state.convolve_max * (
1 + POSITIVE_SEGMENT_MEASUREMENT_ERROR)
lower_bound = self.state.convolve_min * (
1 - POSITIVE_SEGMENT_MEASUREMENT_ERROR)
delete_up_bound = self.state.conv_del_max * (
1 + NEGATIVE_SEGMENT_MEASUREMENT_ERROR)
delete_low_bound = self.state.conv_del_min * (
1 - NEGATIVE_SEGMENT_MEASUREMENT_ERROR)
max_conv = max(conv)
if max_conv > upper_bound or max_conv < lower_bound:
delete_list.append(segment_index)
elif max_conv < delete_up_bound and max_conv > delete_low_bound:
delete_list.append(segment_index)
for item in delete_list:
segments_indexes.remove(item)
segments_indexes = utils.remove_duplicates_and_sort(segments_indexes)
return segments_indexes
def __filter_detection(self, segments: list, data: list) -> list:
delete_list = []
variance_error = self.state['WINDOW_SIZE']
close_patterns = utils.close_filtering(segments, variance_error)
segments = utils.best_pattern(close_patterns, data, 'min')
if len(segments) == 0 or len(self.state.get('pattern_center',
[])) == 0:
segments = []
return segments
pattern_data = self.state['pattern_model']
up_height = self.state['height_max'] * (1 + self.HEIGHT_ERROR)
low_height = self.state['height_min'] * (1 - self.HEIGHT_ERROR)
up_conv = self.state['convolve_max'] * (1 + 1.5 * self.CONV_ERROR)
low_conv = self.state['convolve_min'] * (1 - self.CONV_ERROR)
up_del_conv = self.state['conv_del_max'] * (1 + self.DEL_CONV_ERROR)
low_del_conv = self.state['conv_del_min'] * (1 - self.DEL_CONV_ERROR)
for segment in segments:
if segment > self.state['WINDOW_SIZE']:
convol_data = utils.get_interval(data, segment,
self.state['WINDOW_SIZE'])
convol_data = utils.subtract_min_without_nan(convol_data)
percent_of_nans = convol_data.isnull().sum() / len(convol_data)
if percent_of_nans > 0.5:
delete_list.append(segment)
continue
elif 0 < percent_of_nans <= 0.5:
nan_list = utils.find_nan_indexes(convol_data)
convol_data = utils.nan_to_zero(convol_data, nan_list)
pattern_data = utils.nan_to_zero(pattern_data, nan_list)
conv = scipy.signal.fftconvolve(convol_data, pattern_data)
pattern_height = convol_data.values.max()
if pattern_height > up_height or pattern_height < low_height:
delete_list.append(segment)
continue
if max(conv) > up_conv or max(conv) < low_conv:
delete_list.append(segment)
continue
if max(conv) < up_del_conv and max(conv) > low_del_conv:
delete_list.append(segment)
else:
delete_list.append(segment)
for item in delete_list:
segments.remove(item)
return set(segments)
def __filter_detection(self, segments, data):
delete_list = []
variance_error = self.state['WINDOW_SIZE']
close_patterns = utils.close_filtering(segments, variance_error)
segments = utils.best_pattern(close_patterns, data, 'max')
if len(segments) == 0 or len(self.state.get('pattern_center',
[])) == 0:
segments = []
return segments
pattern_data = self.state['pattern_model']
upper_bound = self.state['convolve_max'] * 1.2
lower_bound = self.state['convolve_min'] * 0.8
delete_up_bound = self.state['conv_del_max'] * 1.02
delete_low_bound = self.state['conv_del_min'] * 0.98
for segment in segments:
if segment > self.state['WINDOW_SIZE'] and segment < (
len(data) - self.state['WINDOW_SIZE']):
convol_data = utils.get_interval(data, segment,
self.state['WINDOW_SIZE'])
percent_of_nans = convol_data.isnull().sum() / len(convol_data)
if len(convol_data) == 0 or percent_of_nans > 0.5:
delete_list.append(segment)
continue
elif 0 < percent_of_nans <= 0.5:
nan_list = utils.find_nan_indexes(convol_data)
convol_data = utils.nan_to_zero(convol_data, nan_list)
pattern_data = utils.nan_to_zero(pattern_data, nan_list)
conv = scipy.signal.fftconvolve(convol_data, pattern_data)
try:
if max(conv) > upper_bound or max(conv) < lower_bound:
delete_list.append(segment)
elif max(conv) < delete_up_bound and max(
conv) > delete_low_bound:
delete_list.append(segment)
except ValueError:
delete_list.append(segment)
else:
delete_list.append(segment)
for item in delete_list:
segments.remove(item)
return set(segments)
def __filter_detection(self, segments: List[int], data: pd.Series) -> list:
delete_list = []
variance_error = self.state.window_size
close_patterns = utils.close_filtering(segments, variance_error)
segments = self.get_best_pattern(close_patterns, data)
if len(segments) == 0 or len(self.state.pattern_model) == 0:
return []
pattern_data = self.state.pattern_model
up_height = self.state.height_max * (1 + self.HEIGHT_ERROR)
low_height = self.state.height_min * (1 - self.HEIGHT_ERROR)
up_conv = self.state.convolve_max * (1 + 1.5 * self.CONV_ERROR)
low_conv = self.state.convolve_min * (1 - self.CONV_ERROR)
up_del_conv = self.state.conv_del_max * (1 + self.DEL_CONV_ERROR)
low_del_conv = self.state.conv_del_min * (1 - self.DEL_CONV_ERROR)
for segment in segments:
if segment > self.state.window_size:
convol_data = utils.get_interval(data, segment, self.state.window_size)
convol_data = utils.subtract_min_without_nan(convol_data)
percent_of_nans = convol_data.isnull().sum() / len(convol_data)
if percent_of_nans > 0.5:
delete_list.append(segment)
continue
elif 0 < percent_of_nans <= 0.5:
nan_list = utils.find_nan_indexes(convol_data)
convol_data = utils.nan_to_zero(convol_data, nan_list)
pattern_data = utils.nan_to_zero(pattern_data, nan_list)
conv = scipy.signal.fftconvolve(convol_data, pattern_data)
pattern_height = convol_data.values.max()
if pattern_height > up_height or pattern_height < low_height:
delete_list.append(segment)
continue
if max(conv) > up_conv or max(conv) < low_conv:
delete_list.append(segment)
continue
if max(conv) < up_del_conv and max(conv) > low_del_conv:
delete_list.append(segment)
else:
delete_list.append(segment)
for item in delete_list:
segments.remove(item)
return set(segments)
def test_find_nan_indexes_empty_values(self):
data = []
result = []
self.assertEqual(utils.find_nan_indexes(data), result)
def test_find_nan_indexes_normal_values(self):
data = [1, 1, 1, 0, 0, 0, 1, 1]
data = pd.Series(data)
result = []
self.assertEqual(utils.find_nan_indexes(data), result)
def test_find_nan_indexes(self):
data = [1, 1, 1, 0, 0, np.nan, None, []]
data = pd.Series(data)
result = [5, 6]
self.assertEqual(utils.find_nan_indexes(data), result)
def convert_nan_to_zero(self):
nan_list = utils.find_nan_indexes(self.data)
self.data = utils.nan_to_zero(self.data, nan_list)
def check_nan_values(
segment: Union[pd.Series, list]) -> Union[pd.Series, list]:
nan_list = utils.find_nan_indexes(segment)
if len(nan_list) > 0:
segment = utils.nan_to_zero(segment, nan_list)
return segment
