def _dezing(self, data): result = data[...] median_result = sig.medfilt(data, self._kernel) differrence = np.abs(data - median_result) replace_mask = differrence > self.parameters['outlier_mu'] self.zinger_proportion = max( self.zinger_proportion, np.sum(replace_mask) / (np.size(replace_mask) * 1.0)) result[replace_mask] = median_result[replace_mask] return result
def _dezing(self, data): result = data[...] median_result = sig.medfilt(data, self._kernel) differrence = np.abs(data-median_result) replace_mask = differrence > self.parameters['outlier_mu'] self.zinger_proportion = \ max(self.zinger_proportion, np.sum(replace_mask)/( np.size(replace_mask)*1.0)) result[replace_mask] = median_result[replace_mask] return result
def filter_frame(self, data): """ The second method we need to implement from the Filter class and the part of the code that actually does all the work. the input here 'data' will contain the 3D block of data to process, and we need to return the data for the single frame in the middle of this. In this case we use the scipy median filter with the 'kernmel_size' parameter, and return the same size data as you had originally. """ logging.debug("Data frame recieved for processing of shape %s", str(data.shape)) result = sig.medfilt(data, self.parameters['kernel_size']) return result
def filter_frames(self, data): result = sig.medfilt(data[0], self.parameters['kernel_size']) return result
def process_frames(self, data): result = sig.medfilt(data[0], self.parameters['kernel_size']) return result
def filt1d(array, nmed, nlin, fill = False, circ = False): """Nonlinear (median+boxcar) filter with edge reflection.""" nmed = 2 * int(nmed/2) + 1 N = len(array) if N < (3*nmed): return array # Check for NaNs lnan = numpy.isnan(array) nnan = sum(lnan) if (nnan != 0): lnotnan = numpy.where(lnan==0) s = numpy.shape(lnotnan) if (len(s)>1): s = numpy.size(lnotnan) lnotnan = numpy.reshape(lnotnan, s) med = numpy.median(array[lnotnan]) # Fill in any data gaps by interpolating over them ... work = numpy.zeros(N) il = numpy.min(lnotnan) ih = numpy.max(lnotnan) xnew = numpy.arange(ih-il+1) + il f = interpolate.interp1d(lnotnan, array[lnotnan]) work[il:ih+1] = f(xnew) # ... or extending slope of nearest data points if at the edges if (il!=0): slope = work[il+1] - work[il] for i in range(il): work[i] = work[il] - slope*(il-i) if (ih!=N-1): slope = work[ih] - work[ih-1] for i in range(N-ih-1)+ih+1: work[i] = work[ih] + slope*(i-ih) else: work = numpy.copy(array) # Edge reflection nr = min(20, nlin) sz = max([nmed, nlin]) if sz >= (N-1): sz = N-2 if circ != False: wl = work[N-sz:] wr = work[:sz] else: wl = array[0:sz] pivot = numpy.median(array[:nr]) wl = 2 * pivot - wl wl = wl[::-1] # reverse array wr = array[N-sz:N] pivot = numpy.median(array[N-nr:N]) wr = 2 * pivot - wr wr = wr[::-1] # reverse array work2 = numpy.zeros(N + 2 * sz) work2[0:sz] = wl work2[sz:sz+N] = work work2[sz+N:2*sz+N] = wr # Filter if nmed > 1: work = signaltools.medfilt(work2, nmed) else: work = work2 box = scipy.signal.boxcar(2*nlin+1) work = signaltools.convolve(work, box) / float(2*nlin+1) padd = (len(work) - N - 2 * sz) / 2 work = work[padd:padd+N+2*sz] # return to orginal array bounds result = work[sz:N+sz] # replace bad data if present if (fill==False) * (nnan!=0): result[lnan] = numpy.nan return result
def filter_frame(self, data, params): logging.debug("Running Filter data") result = sig.medfilt(data, self.parameters['kernel_size']) return result
def filter_frames(self, data): result = sig.medfilt(data[0], self.parameters["kernel_size"]) return result
def filter_frame(self, data): logging.debug("Running Filter data") result = sig.medfilt(data, self.parameters['kernel_size']) return result
def filt1d(array, nmed, nlin, fill=False, circ=False): """Nonlinear (median+boxcar) filter with edge reflection.""" nmed = 2 * int(nmed / 2) + 1 N = len(array) if N < (3 * nmed): return array # Check for NaNs lnan = numpy.isnan(array) nnan = sum(lnan) if (nnan != 0): lnotnan = numpy.where(lnan == 0) s = numpy.shape(lnotnan) if (len(s) > 1): s = numpy.size(lnotnan) lnotnan = numpy.reshape(lnotnan, s) med = numpy.median(array[lnotnan]) # Fill in any data gaps by interpolating over them ... work = numpy.zeros(N) il = numpy.min(lnotnan) ih = numpy.max(lnotnan) xnew = numpy.arange(ih - il + 1) + il f = interpolate.interp1d(lnotnan, array[lnotnan]) work[il:ih + 1] = f(xnew) # ... or extending slope of nearest data points if at the edges if (il != 0): slope = work[il + 1] - work[il] for i in range(il): work[i] = work[il] - slope * (il - i) if (ih != N - 1): slope = work[ih] - work[ih - 1] for i in range(N - ih - 1) + ih + 1: work[i] = work[ih] + slope * (i - ih) else: work = numpy.copy(array) # Edge reflection nr = min(20, nlin) sz = max([nmed, nlin]) if sz >= (N - 1): sz = N - 2 if circ != False: wl = work[N - sz:] wr = work[:sz] else: wl = array[0:sz] pivot = numpy.median(array[:nr]) wl = 2 * pivot - wl wl = wl[::-1] # reverse array wr = array[N - sz:N] pivot = numpy.median(array[N - nr:N]) wr = 2 * pivot - wr wr = wr[::-1] # reverse array work2 = numpy.zeros(N + 2 * sz) work2[0:sz] = wl work2[sz:sz + N] = work work2[sz + N:2 * sz + N] = wr # Filter if nmed > 1: work = signaltools.medfilt(work2, nmed) else: work = work2 box = scipy.signal.boxcar(2 * nlin + 1) work = signaltools.convolve(work, box) / float(2 * nlin + 1) padd = (len(work) - N - 2 * sz) / 2 work = work[padd:padd + N + 2 * sz] # return to orginal array bounds result = work[sz:N + sz] # replace bad data if present if (fill == False) * (nnan != 0): result[lnan] = numpy.nan return result
with open('buffer.txt', 'r') as f: for line in f: if line != '\n' or '': y.append(int(line.strip())) print('File is loaded') """ DETECTOR """ print('Diff launch') y_diff = np.diff(y, 1) # differentiator print('Hilbert launch') y_env = np.abs(hilbert(y_diff)) # Hilbert' filter print('Medfilt launch') m_w = int(len(y) / 30) # Median filter's window y_filtered = medfilt(y_env, m_w + (m_w + 1) % 2) # Median filter print('Norm code') y_filtered = y_filtered / max(y_filtered) # Signal normalization print('Code processing') code = list() for i in range(0, N): if y_filtered[int((1 / (2 * Fbit) + i / Fbit) / (dur / len(y)))] >= 0.75: code.append(1) # High level elif y_filtered[int((1 / (2 * Fbit) + i / Fbit) / (dur / len(y)))] <= 0.55: code.append(0) # low level else: code.append('?') # Undetected print(*code[1:])
def _do_prediction(self, cur_test_df, grid_area, trained_scalers, dimensions, train_df, m, all_feature_sum, features_to_skip, timestamp_gap, expected_rul, fine=-1, dev_dict_path=""): # 1. initialize try: if not all_feature_sum: all_feature_sum = np.empty( grid_area) # resulting curve is sum of subcurves all_feature_sum.fill(0) except: pass # 2. scale - adding column scaled cur_test_df = self._scale_with_scaler(cur_test_df, trained_scalers) feature_favs_dict = {} # 3. Prediction total model if fine == -1: found_risk, found_rul, all_feature_sum, per_feature_sum, feature_favorites, feature_favs_dict = self._do_whole_model_prediction( cur_test_df, grid_area, dimensions, m, all_feature_sum, features_to_skip, timestamp_gap, dev_dict_path) return found_risk, found_rul, all_feature_sum, per_feature_sum, feature_favorites, feature_favs_dict # 3. Prediction side model if fine != -1: weight_b, feature_favorites, per_feature_sum = True, [], {} if self.skip_never: features_to_skip = [] for col in cur_test_df.columns: # 3.1. Initialize info_key = "fine_" + str(fine) + "_" + col if HeatmapConvolutionTrainer.scaled_relevant_columns( col) or col in features_to_skip: continue summ, all_good = self._initialize_feature(grid_area, col) cur_dev = 0 try: l_iter_e = len(cur_test_df[col]) l_iter_s = 0 if l_iter_e > 6: l_iter_s = len(cur_test_df[col]) - 6 l_iter_e = len(cur_test_df[col]) # 3.2. iterate over rows for cur_row in list(range(l_iter_s, l_iter_e)): try: cur_heat_values, xi = self._get_current_heat( cur_test_df, dimensions, cur_row, m, grid_area, col, info_key) except: continue if self._smooth_per_feature: cur_heat_values = medfilt(cur_heat_values, self.smoothing_side) summ = summ + cur_heat_values if self._visualize_pre_post_risk: self._visualize_pre_post_risk_m( xi, cur_model, y_rel_row_idx, cur_heat_values) # 3.3. Shift by bias that was determined bias = cur_dev / m summ, xi = self._shift_array(xi, summ, bias, m, grid_area) # 3.4. Timestamp gap and store summ, xi = self._shift_array(xi, summ, timestamp_gap, m, grid_area) per_feature_sum[col] = deepcopy([xi, cur_heat_values]) #if summ.argmax() == 0: continue #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! all_feature_sum = all_feature_sum + summ if self._visualize_per_feature_curve: self._visualize_per_feature_curve_m(xi, summ) #if self._enable_all_print: print(str(col)+ " - Most-likely risk (Gesamtheit): " + str(xi[np.unravel_index(summ.argmax(), summ.shape)[0]])) if summ.argmax() != 0: feature_favs_dict[col] = xi[np.unravel_index( summ.argmax(), summ.shape )[0]] # WIRD ANDERS BESTIMMT IM ENDERGEBNIS -> WAHL SOLLTE NACH TOP 10 Erfolgen else: pass # 4.2. Variante 2 - top 10 % weighted average ten_per_idx = math.floor( len(all_feature_sum) * self.percentage_side_fine) ind = self.largest_indices(all_feature_sum, ten_per_idx) weight = all_feature_sum[ind] max_val = numpy.max(weight) normalized_weight = weight / max_val weighted_res = numpy.sum( xi[ind] * normalized_weight) / numpy.sum(normalized_weight) found_risk = [xi[ind], normalized_weight] # 4.3. Variante 3 - Maximum of curve feature_favorites.append(xi[np.unravel_index( summ.argmax(), summ.shape)[0]]) except: found_risk = -1 # 5. print results found_rul = -1 try: self._print_likeliness(found_risk, cur_test_df, xi, m, found_rul, expected_rul) if self._visualize_summed_curve: self._visualize_summed_curve_m(xi, all_feature_sum) except: pass return found_risk, found_rul, all_feature_sum, per_feature_sum, feature_favorites, feature_favs_dict
def _do_whole_model_prediction(self, cur_test_df, grid_area, dimensions, m, all_feature_sum, features_to_skip, timestamp_gap, dev_dict_path): # 0 loading deviations if not self.skip_never_whole: try: with open(dev_dict_path, 'rb') as in_strm: dev_dict = dill.load(in_strm) except: dev_dict = {} # 1. Initialize feature_favorites, per_feature_sum, found_risk, weight_b, feature_favs_dict = [], {}, -1, True, {} for col in cur_test_df.columns: if HeatmapConvolutionTrainer.scaled_relevant_columns( col) or col in features_to_skip: continue summ, all_good = self._initialize_feature(grid_area, col) try: l_iter_e = len(cur_test_df[col]) l_iter_s = 0 if l_iter_e > 6: l_iter_s = len(cur_test_df[col]) - 6 l_iter_e = len(cur_test_df[col]) # 2. Per Sample of Training data determine heat curve and add it shifted to most recent this_feature_tops = [] for cur_row in list(range(l_iter_s, l_iter_e)): try: cur_heat_values, xi = self._get_current_heat( cur_test_df, dimensions, cur_row, m, grid_area, col) except: pass #print("John McSkippo") #print(traceback.format_exc()) continue if len(numpy.where(cur_heat_values != 0)[0]) == 0: continue if self._smooth_per_feature: cur_heat_values = medfilt(cur_heat_values, 41) summ = summ + cur_heat_values if self._visualize_pre_post_risk: self._visualize_pre_post_risk_m( xi, cur_model, y_rel_row_idx, cur_heat_values) # 2.1 Gather information per point ten_per_idx = math.floor(len(cur_heat_values) * 0.1) ind = self.largest_indices(cur_heat_values, ten_per_idx) weight = cur_heat_values[ind] max_val = numpy.max(weight) normalized_weight = weight / max_val weighted_res = numpy.sum( xi[ind] * normalized_weight) / numpy.sum(normalized_weight) this_risk = weighted_res this_feature_tops.append(this_risk) avg = numpy.average(this_feature_tops) remaining = this_feature_tops #[f for f in this_feature_tops if (avg+0.2)>f and f>(avg-0.2)] if summ.argmax() != 0: feature_favs_dict[col] = avg try: avg += dev_dict[cur_test_df["cluster_id"].iloc[0]][col] except: pass #3. Timestamp gap: die Cluster sagen nicht alle die selbe RUL voraus muss sie also aufeinander schieben summ, xi = self._shift_array(xi, summ, timestamp_gap, m, grid_area) # 4. store and plot result+ per_feature_sum[col] = deepcopy([xi, cur_heat_values]) all_feature_sum = all_feature_sum + summ if self._visualize_per_feature_curve: self._visualize_per_feature_curve_m(xi, summ) try: two_max = numpy.array(self.find_integrals( xi, summ)).argsort()[-2:][::-1] two_max = two_max.tolist() if 1 in two_max: # vote two_max.remove(1) self.voting[two_max[0]] += 2 self.voting[1] += 1 else: self.voting[two_max[0]] += 1 self.integral_sum = self.find_integrals( xi, all_feature_sum) except: print(traceback.format_exc()) # 5.4. Variante 4: Average of current heats without outliers found_risk = avg if not math.isnan(found_risk): feature_favorites.append(found_risk) except: found_risk = -1 # 6. print results found_rul = (found_risk - 1) / m try: self._print_likeliness(found_risk, cur_test_df, xi, m, found_rul, expected_rul) if self._visualize_summed_curve: self._visualize_summed_curve_m(xi, all_feature_sum) except: pass #print("No") return found_risk, found_rul, all_feature_sum, per_feature_sum, feature_favorites, feature_favs_dict