Exemple #1
0
 def _dezing(self, data):
     result = data[...]
     median_result = sig.medfilt(data, self._kernel)
     differrence = np.abs(data - median_result)
     replace_mask = differrence > self.parameters['outlier_mu']
     self.zinger_proportion = max(
         self.zinger_proportion,
         np.sum(replace_mask) / (np.size(replace_mask) * 1.0))
     result[replace_mask] = median_result[replace_mask]
     return result
 def _dezing(self, data):
     result = data[...]
     median_result = sig.medfilt(data, self._kernel)
     differrence = np.abs(data-median_result)
     replace_mask = differrence > self.parameters['outlier_mu']
     self.zinger_proportion = \
         max(self.zinger_proportion, np.sum(replace_mask)/(
             np.size(replace_mask)*1.0))
     result[replace_mask] = median_result[replace_mask]
     return result
 def filter_frame(self, data):
     """
     The second method we need to implement from the Filter class and the
     part of the code that actually does all the work. the input here 'data'
     will contain the 3D block of data to process, and we need to return the
     data for the single frame in the middle of this. In this case we use
     the scipy median filter with the 'kernmel_size' parameter, and return
     the same size data as you had originally.
     """
     logging.debug("Data frame recieved for processing of shape %s",
                   str(data.shape))
     result = sig.medfilt(data, self.parameters['kernel_size'])
     return result
Exemple #4
0
 def filter_frame(self, data):
     """
     The second method we need to implement from the Filter class and the
     part of the code that actually does all the work. the input here 'data'
     will contain the 3D block of data to process, and we need to return the
     data for the single frame in the middle of this. In this case we use
     the scipy median filter with the 'kernmel_size' parameter, and return
     the same size data as you had originally.
     """
     logging.debug("Data frame recieved for processing of shape %s",
                   str(data.shape))
     result = sig.medfilt(data, self.parameters['kernel_size'])
     return result
Exemple #5
0
 def filter_frames(self, data):
     result = sig.medfilt(data[0], self.parameters['kernel_size'])
     return result
 def process_frames(self, data):
     result = sig.medfilt(data[0], self.parameters['kernel_size'])
     return result
Exemple #7
0
def filt1d(array, nmed, nlin, fill = False, circ = False):
    """Nonlinear (median+boxcar) filter with edge reflection."""
    nmed = 2 * int(nmed/2) + 1
    N = len(array)
    if N < (3*nmed): 
        return array
    # Check for NaNs
    lnan = numpy.isnan(array)
    nnan = sum(lnan)
    if (nnan != 0):
        lnotnan = numpy.where(lnan==0)
        s = numpy.shape(lnotnan)
        if (len(s)>1):
            s = numpy.size(lnotnan)
            lnotnan = numpy.reshape(lnotnan, s)
            med = numpy.median(array[lnotnan])
        # Fill in any data gaps by interpolating over them ...
        work = numpy.zeros(N)
        il = numpy.min(lnotnan)
        ih = numpy.max(lnotnan)
        xnew = numpy.arange(ih-il+1) + il
        f = interpolate.interp1d(lnotnan, array[lnotnan])  
        work[il:ih+1] = f(xnew)
        # ... or extending slope of nearest data points if at the edges
        if (il!=0):
            slope = work[il+1] - work[il]
            for i in range(il): work[i] = work[il] - slope*(il-i)
        if (ih!=N-1):
            slope = work[ih] - work[ih-1]
            for i in range(N-ih-1)+ih+1: work[i] = work[ih] + slope*(i-ih)
    else:
        work = numpy.copy(array)
    # Edge reflection
    nr = min(20, nlin)
    sz = max([nmed, nlin])
    if sz >= (N-1): 
        sz = N-2
    if circ != False:
        wl = work[N-sz:]
        wr = work[:sz]
    else:
        wl = array[0:sz]
        pivot = numpy.median(array[:nr])
        wl = 2 * pivot - wl
        wl = wl[::-1] # reverse array
        wr = array[N-sz:N]
        pivot = numpy.median(array[N-nr:N])
        wr = 2 * pivot - wr
        wr = wr[::-1] # reverse array
    work2 = numpy.zeros(N + 2 * sz)
    work2[0:sz] = wl
    work2[sz:sz+N] = work
    work2[sz+N:2*sz+N] = wr
    # Filter
    if nmed > 1: work = signaltools.medfilt(work2, nmed) 
    else: work = work2
    box = scipy.signal.boxcar(2*nlin+1)
    work = signaltools.convolve(work, box) / float(2*nlin+1)
    padd = (len(work) - N - 2 * sz) / 2
    work = work[padd:padd+N+2*sz]
    # return to orginal array bounds
    result = work[sz:N+sz]
    # replace bad data if present
    if (fill==False) * (nnan!=0): result[lnan] = numpy.nan
    return result
Exemple #8
0
 def filter_frame(self, data, params):
     logging.debug("Running Filter data")
     result = sig.medfilt(data, self.parameters['kernel_size'])
     return result
Exemple #9
0
 def filter_frames(self, data):
     result = sig.medfilt(data[0], self.parameters["kernel_size"])
     return result
Exemple #10
0
 def filter_frame(self, data):
     logging.debug("Running Filter data")
     result = sig.medfilt(data, self.parameters['kernel_size'])
     return result
Exemple #11
0
def filt1d(array, nmed, nlin, fill=False, circ=False):
    """Nonlinear (median+boxcar) filter with edge reflection."""
    nmed = 2 * int(nmed / 2) + 1
    N = len(array)
    if N < (3 * nmed):
        return array
    # Check for NaNs
    lnan = numpy.isnan(array)
    nnan = sum(lnan)
    if (nnan != 0):
        lnotnan = numpy.where(lnan == 0)
        s = numpy.shape(lnotnan)
        if (len(s) > 1):
            s = numpy.size(lnotnan)
            lnotnan = numpy.reshape(lnotnan, s)
            med = numpy.median(array[lnotnan])
        # Fill in any data gaps by interpolating over them ...
        work = numpy.zeros(N)
        il = numpy.min(lnotnan)
        ih = numpy.max(lnotnan)
        xnew = numpy.arange(ih - il + 1) + il
        f = interpolate.interp1d(lnotnan, array[lnotnan])
        work[il:ih + 1] = f(xnew)
        # ... or extending slope of nearest data points if at the edges
        if (il != 0):
            slope = work[il + 1] - work[il]
            for i in range(il):
                work[i] = work[il] - slope * (il - i)
        if (ih != N - 1):
            slope = work[ih] - work[ih - 1]
            for i in range(N - ih - 1) + ih + 1:
                work[i] = work[ih] + slope * (i - ih)
    else:
        work = numpy.copy(array)
    # Edge reflection
    nr = min(20, nlin)
    sz = max([nmed, nlin])
    if sz >= (N - 1):
        sz = N - 2
    if circ != False:
        wl = work[N - sz:]
        wr = work[:sz]
    else:
        wl = array[0:sz]
        pivot = numpy.median(array[:nr])
        wl = 2 * pivot - wl
        wl = wl[::-1]  # reverse array
        wr = array[N - sz:N]
        pivot = numpy.median(array[N - nr:N])
        wr = 2 * pivot - wr
        wr = wr[::-1]  # reverse array
    work2 = numpy.zeros(N + 2 * sz)
    work2[0:sz] = wl
    work2[sz:sz + N] = work
    work2[sz + N:2 * sz + N] = wr
    # Filter
    if nmed > 1: work = signaltools.medfilt(work2, nmed)
    else: work = work2
    box = scipy.signal.boxcar(2 * nlin + 1)
    work = signaltools.convolve(work, box) / float(2 * nlin + 1)
    padd = (len(work) - N - 2 * sz) / 2
    work = work[padd:padd + N + 2 * sz]
    # return to orginal array bounds
    result = work[sz:N + sz]
    # replace bad data if present
    if (fill == False) * (nnan != 0): result[lnan] = numpy.nan
    return result
Exemple #12
0
with open('buffer.txt', 'r') as f:
    for line in f:
        if line != '\n' or '':
            y.append(int(line.strip()))

print('File is loaded')
"""
DETECTOR
"""
print('Diff launch')
y_diff = np.diff(y, 1)  # differentiator
print('Hilbert launch')
y_env = np.abs(hilbert(y_diff))  # Hilbert' filter
print('Medfilt launch')
m_w = int(len(y) / 30)  # Median filter's window
y_filtered = medfilt(y_env, m_w + (m_w + 1) % 2)  # Median filter
print('Norm code')
y_filtered = y_filtered / max(y_filtered)  # Signal normalization
print('Code processing')

code = list()
for i in range(0, N):
    if y_filtered[int((1 / (2 * Fbit) + i / Fbit) / (dur / len(y)))] >= 0.75:
        code.append(1)  # High level
    elif y_filtered[int((1 / (2 * Fbit) + i / Fbit) / (dur / len(y)))] <= 0.55:
        code.append(0)  # low level
    else:
        code.append('?')  # Undetected
print(*code[1:])
    def _do_prediction(self,
                       cur_test_df,
                       grid_area,
                       trained_scalers,
                       dimensions,
                       train_df,
                       m,
                       all_feature_sum,
                       features_to_skip,
                       timestamp_gap,
                       expected_rul,
                       fine=-1,
                       dev_dict_path=""):
        # 1. initialize
        try:
            if not all_feature_sum:
                all_feature_sum = np.empty(
                    grid_area)  # resulting curve is sum of subcurves
                all_feature_sum.fill(0)
        except:
            pass

        # 2. scale - adding column scaled
        cur_test_df = self._scale_with_scaler(cur_test_df, trained_scalers)
        feature_favs_dict = {}

        # 3. Prediction total model
        if fine == -1:
            found_risk, found_rul, all_feature_sum, per_feature_sum, feature_favorites, feature_favs_dict = self._do_whole_model_prediction(
                cur_test_df, grid_area, dimensions, m, all_feature_sum,
                features_to_skip, timestamp_gap, dev_dict_path)

            return found_risk, found_rul, all_feature_sum, per_feature_sum, feature_favorites, feature_favs_dict

        # 3. Prediction side model
        if fine != -1:
            weight_b, feature_favorites, per_feature_sum = True, [], {}
            if self.skip_never: features_to_skip = []

            for col in cur_test_df.columns:
                # 3.1. Initialize
                info_key = "fine_" + str(fine) + "_" + col
                if HeatmapConvolutionTrainer.scaled_relevant_columns(
                        col) or col in features_to_skip:
                    continue
                summ, all_good = self._initialize_feature(grid_area, col)
                cur_dev = 0

                try:
                    l_iter_e = len(cur_test_df[col])
                    l_iter_s = 0
                    if l_iter_e > 6:
                        l_iter_s = len(cur_test_df[col]) - 6
                        l_iter_e = len(cur_test_df[col])

                    # 3.2. iterate over rows
                    for cur_row in list(range(l_iter_s, l_iter_e)):

                        try:
                            cur_heat_values, xi = self._get_current_heat(
                                cur_test_df, dimensions, cur_row, m, grid_area,
                                col, info_key)
                        except:
                            continue
                        if self._smooth_per_feature:
                            cur_heat_values = medfilt(cur_heat_values,
                                                      self.smoothing_side)
                        summ = summ + cur_heat_values
                        if self._visualize_pre_post_risk:
                            self._visualize_pre_post_risk_m(
                                xi, cur_model, y_rel_row_idx, cur_heat_values)

                    # 3.3. Shift by bias that was determined
                    bias = cur_dev / m
                    summ, xi = self._shift_array(xi, summ, bias, m, grid_area)

                    # 3.4. Timestamp gap and store
                    summ, xi = self._shift_array(xi, summ, timestamp_gap, m,
                                                 grid_area)
                    per_feature_sum[col] = deepcopy([xi, cur_heat_values])
                    #if summ.argmax() == 0: continue #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
                    all_feature_sum = all_feature_sum + summ
                    if self._visualize_per_feature_curve:
                        self._visualize_per_feature_curve_m(xi, summ)

                    #if self._enable_all_print: print(str(col)+ " - Most-likely risk (Gesamtheit): " + str(xi[np.unravel_index(summ.argmax(), summ.shape)[0]]))
                    if summ.argmax() != 0:
                        feature_favs_dict[col] = xi[np.unravel_index(
                            summ.argmax(), summ.shape
                        )[0]]  # WIRD ANDERS BESTIMMT IM ENDERGEBNIS -> WAHL SOLLTE NACH TOP 10 Erfolgen
                    else:
                        pass

                    # 4.2. Variante 2 - top 10 % weighted average
                    ten_per_idx = math.floor(
                        len(all_feature_sum) * self.percentage_side_fine)
                    ind = self.largest_indices(all_feature_sum, ten_per_idx)
                    weight = all_feature_sum[ind]
                    max_val = numpy.max(weight)
                    normalized_weight = weight / max_val
                    weighted_res = numpy.sum(
                        xi[ind] *
                        normalized_weight) / numpy.sum(normalized_weight)
                    found_risk = [xi[ind], normalized_weight]

                    # 4.3. Variante 3 - Maximum of curve
                    feature_favorites.append(xi[np.unravel_index(
                        summ.argmax(), summ.shape)[0]])

                except:
                    found_risk = -1

            # 5. print results
            found_rul = -1
            try:
                self._print_likeliness(found_risk, cur_test_df, xi, m,
                                       found_rul, expected_rul)
                if self._visualize_summed_curve:
                    self._visualize_summed_curve_m(xi, all_feature_sum)
            except:
                pass

            return found_risk, found_rul, all_feature_sum, per_feature_sum, feature_favorites, feature_favs_dict
    def _do_whole_model_prediction(self, cur_test_df, grid_area, dimensions, m,
                                   all_feature_sum, features_to_skip,
                                   timestamp_gap, dev_dict_path):

        # 0 loading deviations
        if not self.skip_never_whole:
            try:
                with open(dev_dict_path, 'rb') as in_strm:
                    dev_dict = dill.load(in_strm)
            except:
                dev_dict = {}

        # 1. Initialize
        feature_favorites, per_feature_sum, found_risk, weight_b, feature_favs_dict = [], {}, -1, True, {}

        for col in cur_test_df.columns:
            if HeatmapConvolutionTrainer.scaled_relevant_columns(
                    col) or col in features_to_skip:
                continue
            summ, all_good = self._initialize_feature(grid_area, col)

            try:

                l_iter_e = len(cur_test_df[col])
                l_iter_s = 0
                if l_iter_e > 6:
                    l_iter_s = len(cur_test_df[col]) - 6
                    l_iter_e = len(cur_test_df[col])

                # 2. Per Sample of Training data determine heat curve and add it shifted to most recent
                this_feature_tops = []
                for cur_row in list(range(l_iter_s, l_iter_e)):
                    try:
                        cur_heat_values, xi = self._get_current_heat(
                            cur_test_df, dimensions, cur_row, m, grid_area,
                            col)
                    except:
                        pass  #print("John McSkippo")
                        #print(traceback.format_exc())
                        continue

                    if len(numpy.where(cur_heat_values != 0)[0]) == 0: continue
                    if self._smooth_per_feature:
                        cur_heat_values = medfilt(cur_heat_values, 41)
                    summ = summ + cur_heat_values
                    if self._visualize_pre_post_risk:
                        self._visualize_pre_post_risk_m(
                            xi, cur_model, y_rel_row_idx, cur_heat_values)

                    # 2.1 Gather information per point
                    ten_per_idx = math.floor(len(cur_heat_values) * 0.1)
                    ind = self.largest_indices(cur_heat_values, ten_per_idx)
                    weight = cur_heat_values[ind]
                    max_val = numpy.max(weight)
                    normalized_weight = weight / max_val
                    weighted_res = numpy.sum(
                        xi[ind] *
                        normalized_weight) / numpy.sum(normalized_weight)
                    this_risk = weighted_res
                    this_feature_tops.append(this_risk)

                avg = numpy.average(this_feature_tops)
                remaining = this_feature_tops  #[f for f in this_feature_tops if (avg+0.2)>f and f>(avg-0.2)]

                if summ.argmax() != 0:
                    feature_favs_dict[col] = avg
                try:
                    avg += dev_dict[cur_test_df["cluster_id"].iloc[0]][col]
                except:
                    pass

                #3. Timestamp gap: die Cluster sagen nicht alle die selbe RUL voraus muss sie also aufeinander schieben
                summ, xi = self._shift_array(xi, summ, timestamp_gap, m,
                                             grid_area)

                # 4. store and plot result+
                per_feature_sum[col] = deepcopy([xi, cur_heat_values])
                all_feature_sum = all_feature_sum + summ
                if self._visualize_per_feature_curve:
                    self._visualize_per_feature_curve_m(xi, summ)

                try:
                    two_max = numpy.array(self.find_integrals(
                        xi, summ)).argsort()[-2:][::-1]
                    two_max = two_max.tolist()
                    if 1 in two_max:
                        # vote
                        two_max.remove(1)
                        self.voting[two_max[0]] += 2
                        self.voting[1] += 1
                    else:
                        self.voting[two_max[0]] += 1
                    self.integral_sum = self.find_integrals(
                        xi, all_feature_sum)
                except:
                    print(traceback.format_exc())

                # 5.4. Variante 4: Average of current heats without outliers
                found_risk = avg
                if not math.isnan(found_risk):
                    feature_favorites.append(found_risk)

            except:
                found_risk = -1

        # 6. print results
        found_rul = (found_risk - 1) / m
        try:
            self._print_likeliness(found_risk, cur_test_df, xi, m, found_rul,
                                   expected_rul)
            if self._visualize_summed_curve:
                self._visualize_summed_curve_m(xi, all_feature_sum)
        except:
            pass  #print("No")

        return found_risk, found_rul, all_feature_sum, per_feature_sum, feature_favorites, feature_favs_dict